From 215741b8cd357246cee7c69c01cf678575ef3197 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 15:31:52 -0700
Subject: [PATCH 001/173] fix: decode reordered comprehension restores in
 loaded bytecode

CPython 3.14's optimizer can store an inlined comprehension's result
before restoring the saved local (POP_ITER; STORE_FAST result;
STORE_FAST k). The decoder only ended a shared LOCAL|CELL slot's
plain-local region at a store after SWAP, so a later comprehension's
closure over the same name decoded as a plain local load and raised
UnboundLocalError once the module loaded from its .pyc. The region
now also ends at a store in the run of stores right after the loop's
END_FOR/POP_ITER.
---
 crates/weavepy-compiler/src/cpython_code.rs | 17 +++++++++++++++--
 tests/regrtest/test_pyc_roundtrip.py        | 16 ++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/crates/weavepy-compiler/src/cpython_code.rs b/crates/weavepy-compiler/src/cpython_code.rs
index 8109641a..266edc81 100644
--- a/crates/weavepy-compiler/src/cpython_code.rs
+++ b/crates/weavepy-compiler/src/cpython_code.rs
@@ -2144,8 +2144,13 @@ fn decode_instructions(
     // A comprehension whose own target is a cell announces itself with
     // `MAKE_CELL s` right after the save and keeps closure semantics.
     // Nested comprehensions re-binding the same name push the slot
-    // again, so each restore pops one entry.
+    // again, so each restore pops one entry. The restore usually
+    // follows a `SWAP`, but the optimizer may move the result's store
+    // ahead of it (`POP_ITER; STORE_FAST result; STORE_FAST s`), so a
+    // store in the run of stores and swaps right after the loop's
+    // `END_FOR`/`POP_ITER` ends the region too.
     let mut plain_shared: Vec<u32> = Vec::new();
+    let mut loop_tail = false;
     let shared_local = |slot: u32| slot < slots.nlocals && slots.is_cellish(slot);
     let end_region = |plain_shared: &mut Vec<u32>, slot: u32| {
         if let Some(p) = plain_shared.iter().rposition(|&s| s == slot) {
@@ -2153,7 +2158,15 @@ fn decode_instructions(
         }
     };
     for (idx, r) in raws.iter().enumerate() {
-        let after_swap = idx > 0 && raws[idx - 1].cp_op == op::SWAP;
+        let in_tail = loop_tail;
+        loop_tail = match r.cp_op {
+            op::END_FOR | op::POP_ITER => true,
+            op::SWAP | op::STORE_FAST | op::STORE_FAST_STORE_FAST | op::POP_TOP | op::NOP => {
+                loop_tail
+            }
+            _ => false,
+        };
+        let after_swap = in_tail || (idx > 0 && raws[idx - 1].cp_op == op::SWAP);
         match r.cp_op {
             op::LOAD_FAST_AND_CLEAR
                 if shared_local(r.arg)
diff --git a/tests/regrtest/test_pyc_roundtrip.py b/tests/regrtest/test_pyc_roundtrip.py
index a83117a0..370ab165 100644
--- a/tests/regrtest/test_pyc_roundtrip.py
+++ b/tests/regrtest/test_pyc_roundtrip.py
@@ -72,4 +72,20 @@ def code_to_timestamp_pyc(code, mtime=0, source_size=0):
     assert ns2["VALUE"] == 37
     assert ns2["compute"](3) == 10
 
+# ---------- an inlined comprehension's name, later a closure cell ----------
+# `k` is a plain local of the first comprehension and a cell of the
+# second; the optimizer stores the first's result before restoring `k`
+# (`POP_ITER; STORE_FAST counts; STORE_FAST k`), so the decoder must end
+# the first comprehension's region there, or the second's closure reads
+# the plain local.
+src = (
+    "def summarize(widgets, kinds):\n"
+    "    counts = {k: len(v) for k, v in widgets.items()}\n"
+    "    found = {k: sum(1 for w in widgets if w is k) for k in kinds}\n"
+    "    return counts, found\n"
+)
+ns3 = {}
+exec(marshal.loads(marshal.dumps(compile(src, "<pyc-test>", "exec"))), ns3)
+assert ns3["summarize"]({1: [0], 2: []}, [1, 3]) == ({1: 1, 2: 0}, {1: 1, 3: 0})
+
 print("test_pyc_roundtrip: OK")

From 06f4e6dff26fbaedc8fc2b9b423b7fb697714131 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 16:08:24 -0700
Subject: [PATCH 002/173] perf: allocate bare instances from the cached plan

A class whose memoized instance plan is the bare allocation (the default
__new__ and object.__init__, no native payload) now constructs C() from a
guard-free read of the plan and moves the class off the stack, instead of
re-deriving the shape, walking the MRO twice for module and generic-alias
bases, and cloning the class and plan. The plan records the verdict as
bare_alloc.

Recycling a pooled instance no longer rebuilds slot storage that is
already empty, and resetting an instance's split values stages them on
the stack instead of allocating a vector to drop them.

A generator body whose loop yields now compiles from a back edge's heat,
as it already did from a step's: its yield ends a resume whether or not
the body runs natively.

Empty-class construction retires 22% fewer instructions per instance,
and one storing an attribute 20% fewer.
---
 crates/weavepy-vm/src/frame_jit.rs |  7 ++--
 crates/weavepy-vm/src/inst_dict.rs | 31 ++++++++++++++++-
 crates/weavepy-vm/src/lib.rs       | 54 +++++++++++++++++++++++-------
 crates/weavepy-vm/src/types.rs     | 41 +++++++++++++++++++++--
 4 files changed, 114 insertions(+), 19 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index a788b681..50032f71 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -515,9 +515,10 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
                     ins_k.op,
                     OpCode::EndFor | OpCode::PopIter | OpCode::ReturnValue
                 ) => {}
-                // A generator step ends at its yield whether or not the
-                // body runs natively: no round trip to pay for.
-                None if ins_k.op == OpCode::YieldValue && matches!(at, Heat::Step) => {}
+                // A resume ends at its yield whether or not the body runs
+                // natively: no round trip to pay for.
+                None if ins_k.op == OpCode::YieldValue
+                    && (matches!(at, Heat::Step) || code.is_generator) => {}
                 None => exits += 1,
             }
         }
diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index 5485a91e..b8c2b35b 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -664,8 +664,37 @@ impl SplitValues {
     }
 
     /// Drop every value, keeping the allocation, and forget the names.
+    ///
+    /// The values leave the block before any of them drops (a drop can run
+    /// code that reaches this instance), staged on the stack when there
+    /// are few rather than in a fresh vector.
     pub fn reset(&mut self) {
-        drop(self.take());
+        const STAGE: usize = 8;
+        let Some(b) = self.block else {
+            return;
+        };
+        let h = b.as_ptr();
+        // SAFETY: as `take`: the first `len` values move out exactly once
+        // (the length is zeroed before anything can observe it), and the
+        // names' strong count is released once.
+        unsafe {
+            let n = (*h).len as usize;
+            if n > STAGE {
+                drop(self.take());
+                return;
+            }
+            let mut staged = [const { std::mem::MaybeUninit::<Object>::uninit() }; STAGE];
+            std::ptr::copy_nonoverlapping(Self::values_ptr(h), staged.as_mut_ptr().cast(), n);
+            (*h).len = 0;
+            let keys = std::mem::replace(&mut (*h).keys, std::ptr::null());
+            if !keys.is_null() {
+                drop(Rc::from_raw(keys));
+            }
+            std::ptr::drop_in_place(std::ptr::slice_from_raw_parts_mut(
+                staged.as_mut_ptr().cast::<Object>(),
+                n,
+            ));
+        }
     }
 }
 
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 5228eaa8..438f09ce 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -16467,6 +16467,36 @@ impl Interpreter {
         {
             return false;
         }
+        // `C()` of a class whose memoized plan is the bare allocation: the
+        // class moves off the stack (the self slot owns nothing) and the
+        // instance takes its place.
+        if argc == 0
+            && ty.native_kind.get() == 0
+            // SAFETY: a guard-free read of the memoized plan; nothing runs
+            // before its last use.
+            && unsafe { ty.instance_plan.peek() }
+                .and_then(Option::as_ref)
+                .is_some_and(|(v, plan)| *v == ty.attr_version.get() && plan.bare_alloc)
+        {
+            // SAFETY: the class's slot is initialized and leaves the stack
+            // with the (`Unbound`) self slot, which has no drop glue.
+            let ty = unsafe {
+                let ty = frame.stack.as_ptr().add(callee_slot).read();
+                frame.stack.set_len(callee_slot);
+                ty
+            };
+            let Object::Type(cls) = &ty else {
+                unreachable!("checked above")
+            };
+            let (inst, tracked) = self.alloc_plain_instance_obj(cls);
+            frame.stack.push(inst);
+            frame.pc = pc as u32 + 1;
+            drop(ty);
+            if tracked && gc_trace::maybe_auto_collect() {
+                self.run_pending_finalizers();
+            }
+            return true;
+        }
         let ty = ty.clone();
         let plan = self.instance_plan(&ty);
         // A named tuple: the tuple of the arguments is the whole call (see
@@ -16511,19 +16541,7 @@ impl Interpreter {
             }
             // `C()` for a class with the default `__new__` and
             // `object.__init__`: the allocation is the whole call.
-            if argc == 0
-                && plan.is_object_new
-                && plan.init_from_object
-                && plan.user_new.is_none()
-                && plan.abstract_error.is_none()
-                && !plan.seeds_exception_args
-                && matches!(plan.native, crate::types::NativeKind::Plain)
-                && ty.native_kind.get() == 0
-            {
-                let bt = builtin_types();
-                if ty.is_subclass_of(&bt.module_) || ty.is_subclass_of(&bt.generic_alias_) {
-                    return false;
-                }
+            if argc == 0 && plan.bare_alloc && ty.native_kind.get() == 0 {
                 let (inst, tracked) = self.alloc_plain_instance_obj(&ty);
                 frame.stack.truncate(callee_slot);
                 frame.stack.push(inst);
@@ -53861,6 +53879,15 @@ impl Interpreter {
                 _ => false,
             };
 
+        let bare_alloc = is_object_new
+            && init_from_object
+            && user_new.is_none()
+            && abstract_error.is_none()
+            && !seeds_exception_args
+            && matches!(native, NativeKind::Plain)
+            && !cls.is_subclass_of(&bt.module_)
+            && !cls.is_subclass_of(&bt.generic_alias_);
+
         crate::types::InstancePlan {
             abstract_error,
             user_new,
@@ -53873,6 +53900,7 @@ impl Interpreter {
             only_object_init,
             lean_init,
             tuple_new,
+            bare_alloc,
         }
     }
 
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index f3cc4009..d98a4276 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -858,6 +858,11 @@ pub struct InstancePlan {
     /// its field count. A call with exactly that many positional arguments builds
     /// the tuple directly while the function still runs that code.
     pub tuple_new: Option<(usize, usize)>,
+    /// `C()` with no arguments is the allocation alone: the default
+    /// `__new__` and `object.__init__`, no native payload, not abstract,
+    /// and not a module or generic-alias subclass (whose allocators
+    /// differ).
+    pub bare_alloc: bool,
 }
 
 /// How a fresh instance's `native` payload is provisioned (see
@@ -1245,6 +1250,17 @@ impl TypeObject {
         }
     }
 
+    /// Make `slots` this class's [`Self::fresh_slots`], leaving storage
+    /// that is already the empty default alone.
+    #[inline]
+    pub(crate) fn reset_slots(&self, slots: &mut SlotStorage) {
+        match self.slot_layout.get_or_init(|| self.member_slot_layout()) {
+            Some(layout) => *slots = SlotStorage::unset_over(layout.clone()),
+            None if slots.is_empty_default() => {}
+            None => *slots = SlotStorage::default(),
+        }
+    }
+
     /// Whether the class is one whose native method bodies read its slots
     /// by the position its `__init__` sets them in (`collections.deque`,
     /// `contextvars.Context`, asyncio's handles and futures): those keep
@@ -2290,6 +2306,19 @@ impl SlotStorage {
         }
     }
 
+    /// Whether this is the empty [`Default`] storage, which a reset to the
+    /// default would only rebuild (through the enum's drop glue).
+    #[inline(always)]
+    pub(crate) fn is_empty_default(&self) -> bool {
+        matches!(
+            self.data,
+            SlotData::Single {
+                key: None,
+                value: Object::None
+            }
+        )
+    }
+
     /// Storage holding a natively packed value (see [`SlotData::Packed`]).
     pub(crate) fn from_packed(packed: crate::stdlib::datetime_native::PackedSlots) -> Self {
         Self {
@@ -2378,6 +2407,12 @@ impl SlotStorage {
         Self::default()
     }
 
+    /// Never skipped here (see the 64-bit variant).
+    #[inline(always)]
+    pub(crate) fn is_empty_default(&self) -> bool {
+        false
+    }
+
     /// [`Self::index_of`] (never laid out here; see the 64-bit variant).
     pub fn store_position(&self, name: &str) -> Option<u32> {
         self.index_of(name)
@@ -3107,7 +3142,7 @@ impl PyInstance {
             // class changes. A pooled instance keeps whatever dict it
             // was retired with, already cleared and carrying the record.
             if let Some(m) = Rc::get_mut(&mut inst) {
-                *m.slots.get_mut() = class.fresh_slots();
+                class.reset_slots(m.slots.get_mut());
                 *m.class.get_mut() = class;
                 m.deferred.set(true);
                 if hint > 0 {
@@ -3211,7 +3246,9 @@ impl PyInstance {
             d.map_mut_atomic_store().clear();
             d.reset_deferred_owner(owner);
         }
-        *m.slots.get_mut() = SlotStorage::default();
+        if !m.slots.get_mut().is_empty_default() {
+            *m.slots.get_mut() = SlotStorage::default();
+        }
         m.inline_values.set(true);
         m.deferred.set(true);
         m.hash_cache = crate::sync::CachedHash::new(None);

From 712b1f88ccf4344313ebc6af44b57b06b3f212fd Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 17:29:30 -0700
Subject: [PATCH 003/173] perf: call functions that take defaults directly

A compiled function called with fewer positional arguments than it
declares, its positional defaults filling the rest, now binds through
the frame JIT's direct call and the core loop's plain call, as an
exact-arity call already did. The site records how many defaults it
fills, and a function whose __defaults__ was rebound takes the general
path. deepcopy(x, memo) and go's find() take this path.

The frame JIT's compile verdict counts a CALL or CALL_KW as an
instruction it runs through a helper rather than as an exit, since
call_op runs calls without leaving native code, and an exhausted tuple
iterator that alone holds its tuple's second reference leaves the stack
natively, as a list's already did, instead of handing FOR_ITER to the
core loop once per loop.

Instructions retired against the previous commit: go -4.7%, nqueens
-4.2%, deepcopy -2.5%, raytrace -2.0%, chaos -1.5%; the rest within
the quick build's codegen noise.
---
 crates/weavepy-vm/src/frame_jit.rs | 57 +++++++++++++++++++++---------
 crates/weavepy-vm/src/lib.rs       | 33 ++++++++++++++---
 2 files changed, 68 insertions(+), 22 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 50032f71..6a500610 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -425,7 +425,10 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
     // a round trip.
     let gain = |k: usize| -> Option<usize> {
         let op = ins[k].op;
-        if helped[k] {
+        // A call runs through its helpers (`call_op`), as the core loop's
+        // arm runs it: a compiled callee is called directly, any other
+        // activation switched to without a round trip through the loop.
+        if helped[k] || matches!(op, OpCode::Call | OpCode::CallKw) {
             return Some(1);
         }
         if !native_at(code, k) {
@@ -1122,13 +1125,21 @@ unsafe extern "C" fn h_for_iter(st: *mut State, it: *mut Object, out: *mut Objec
                 None => return 3,
             }
         }
-        PyIterator::Tuple { items, index } => {
-            let Some(v) = items.get(*index).cloned() else {
-                return 3;
-            };
-            *index += 1;
-            v
-        }
+        PyIterator::Tuple { items, index } => match items.get(*index) {
+            Some(v) => {
+                let v = v.clone();
+                *index += 1;
+                v
+            }
+            // The exhausted iterator leaves the stack, as for a list: its
+            // tuple has another owner, so nothing dies with it.
+            None if unique && crate::object::SharedTuple::strong_count(items) > 1 => {
+                // SAFETY: as above.
+                drop(unsafe { it.read() });
+                return 2;
+            }
+            None => return 3,
+        },
         // `reversed(xs)`: the item below the cursor.
         PyIterator::Reversed { items, index, .. } => {
             if *index < 0 {
@@ -2391,6 +2402,8 @@ pub(crate) struct DirectSite {
     code: usize,
     native: usize,
     retry: u32,
+    /// How many trailing parameters take the function's defaults.
+    missing: u32,
     has_self: bool,
 }
 
@@ -2409,7 +2422,7 @@ impl Drop for DirectSite {
 impl DirectSite {
     /// Record a call of `code` (with or without its self slot): its native
     /// body if it qualifies, or none until `retry` more calls.
-    fn fill(&mut self, code: &Rc<CodeObject>, has_self: bool, native: usize) {
+    fn fill(&mut self, code: &Rc<CodeObject>, has_self: bool, native: usize, missing: usize) {
         if Rc::as_ptr(code) as usize != self.code {
             let old = std::mem::replace(&mut self.code, Rc::into_raw(code.clone()) as usize);
             if old != 0 {
@@ -2418,6 +2431,7 @@ impl DirectSite {
             }
         }
         self.has_self = has_self;
+        self.missing = missing as u32;
         self.native = native;
         self.retry = if native == 0 { 64 } else { 0 };
     }
@@ -2425,12 +2439,14 @@ impl DirectSite {
 
 /// Whether a call of `f` (running `code`) with `nargs` positional
 /// arguments binds as [`Interpreter::core_call_plain`] binds it, and
-/// `code`'s native body runs from its start: that body.
+/// `code`'s native body runs from its start: that body, and how many
+/// trailing parameters take `f`'s defaults.
 fn direct_body<'a>(
     f: &crate::object::PyFunction,
     code: &'a CodeObject,
     nargs: usize,
-) -> Option<&'a Native> {
+) -> Option<(&'a Native, usize)> {
+    let missing = (code.arg_count as usize).checked_sub(nargs)?;
     if code.is_generator
         || code.is_coroutine
         || code.is_async_generator
@@ -2438,7 +2454,7 @@ fn direct_body<'a>(
         || !code.freevars.is_empty()
         || !f.closure.is_empty()
         || Interpreter::has_extended_params(code)
-        || code.arg_count as usize != nargs
+        || (missing > 0 && !Interpreter::defaults_cover(f, missing))
         || code.wire.as_ref().is_some_and(|w| w.exec_error.is_some())
     {
         return None;
@@ -2447,7 +2463,7 @@ fn direct_body<'a>(
     if ext.dispatch_len == 0 {
         return None;
     }
-    ext.frame_jit.get(code.varnames.len().max(nargs))
+    Some((ext.frame_jit.get(code.varnames.len().max(nargs))?, missing))
 }
 
 /// `BINARY_OP` (the `pc`th instruction of `code`) over the instances at
@@ -2701,13 +2717,20 @@ unsafe fn direct_call(
             || site.native == 0
         {
             let nargs = len - start - 2 + usize::from(has_self);
-            let native = direct_body(f, code_rc, nargs)
-                .map_or(0, |n| std::ptr::from_ref::<Native>(n) as usize);
-            site.fill(code_rc, has_self, native);
+            let (native, missing) = direct_body(f, code_rc, nargs).map_or((0, 0), |(n, m)| {
+                (std::ptr::from_ref::<Native>(n) as usize, m)
+            });
+            site.fill(code_rc, has_self, native, missing);
             if native == 0 {
                 return None;
             }
         }
+        // (Another function running the same code may have rebound its
+        // defaults.)
+        let missing = site.missing as usize;
+        if missing > 0 && !Interpreter::defaults_cover(f, missing) {
+            return None;
+        }
         let native = &*(site.native as *const Native);
         let interp = &mut *st.interp.cast_mut();
         let nested = interp.direct_calls.get();
@@ -2736,7 +2759,7 @@ unsafe fn direct_call(
         caller.pc = pc as u32;
         *sw.last = st.last;
         let saved_last = sw.last;
-        let callee = interp.core_bind_plain(sw, pc, start, has_self, fp, act, guard);
+        let callee = interp.core_bind_plain(sw, pc, start, has_self, missing, fp, act, guard);
         crate::burst_stats::note_call(crate::burst_stats::CALL_DIRECT);
         let depth = (*sw.inl).len();
         if let Some(a) = (*sw.inl).last_mut() {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 438f09ce..6593d48f 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -15179,7 +15179,13 @@ impl Interpreter {
         let Some(slot) = code_call_slot(&frame.code, pc) else {
             return false;
         };
-        if slot.hit(fp, Rc::as_ptr(code_rc)) != Some((0, has_self))
+        let Some((missing, true)) = slot
+            .hit(fp, Rc::as_ptr(code_rc))
+            .map(|(missing, s)| (missing as usize, s == has_self))
+        else {
+            return false;
+        };
+        if (missing > 0 && !Self::defaults_cover(f, missing))
             || !code_rc.cellvars.is_empty()
             || !code_rc.freevars.is_empty()
             || !f.closure.is_empty()
@@ -15197,21 +15203,30 @@ impl Interpreter {
             return false;
         };
         // SAFETY: as above; the checks above hold.
-        unsafe { self.core_bind_plain(sw, pc, callee_slot, has_self, fp, act, guard) };
+        unsafe { self.core_bind_plain(sw, pc, callee_slot, has_self, missing, fp, act, guard) };
         true
     }
 
+    /// Whether `f`'s compiled positional defaults (`__defaults__` never
+    /// rebound) cover the last `missing` parameters.
+    #[inline]
+    fn defaults_cover(f: &PyFunction, missing: usize) -> bool {
+        f.defaults.len() >= missing && !f.defaults_maybe_overridden()
+    }
+
     /// The committed half of [`Self::core_call_plain`]: the arguments of
     /// the `CALL` at `pc` of `sw`'s running activation (the callable at
     /// `callee_slot`, its self slot filled when `has_self`) move into the
     /// pooled slot `act`'s locals, and `act`, bound to the plain function
     /// `fp` (with exactly its arity) under the recursion `guard`, is
-    /// pushed and made the running activation.
+    /// pushed and made the running activation. The last `missing`
+    /// parameters take `fp`'s positional defaults.
     ///
     /// # Safety
     ///
     /// The running activation is synced and unborrowed, and the call's
-    /// shape is one `core_call_plain` admits.
+    /// shape is one `core_call_plain` admits (the defaults cover
+    /// `missing`, see [`Self::defaults_cover`]).
     #[inline(always)]
     #[allow(clippy::too_many_arguments)]
     unsafe fn core_bind_plain(
@@ -15220,6 +15235,7 @@ impl Interpreter {
         pc: usize,
         callee_slot: usize,
         has_self: bool,
+        missing: usize,
         fp: *const crate::object::PyFunction,
         mut act: Box<InlineAct>,
         guard: crate::recursion::Guard,
@@ -15242,7 +15258,7 @@ impl Interpreter {
             callee_slot + 2
         };
         let nargs = n - first;
-        locals.reserve(nlocals.max(nargs));
+        locals.reserve(nlocals.max(nargs + missing));
         // SAFETY: the `nargs` operands above `first` move into the locals
         // (reserved above) and the stack forgets them; the callable moves
         // out of its slot, and an empty self slot owns nothing.
@@ -15256,6 +15272,13 @@ impl Interpreter {
             frame.stack.set_len(callee_slot);
             callable
         };
+        if missing > 0 {
+            // SAFETY: as above.
+            let defaults = unsafe { &(*fp).defaults };
+            for d in &defaults[defaults.len() - missing..] {
+                push_fast(locals, clone_hot(d));
+            }
+        }
         fill_unbound(locals, nlocals);
         frame.pc = pc as u32 + 1;
         // SAFETY: the slot is parked (see `inline_bind_cells`): its handles

From 0049ed53b70e3675194080fc62c59fed7b1cd41c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 18:34:59 -0700
Subject: [PATCH 004/173] perf: construct store-only __init__ classes by their
 stores

Most __init__ methods only store their parameters (or constants) into
self and return None. Such a body's shape is recognized once per code
object, and a class whose __init__ has it constructs an instance by
those stores alone: the arguments move straight into the fresh
instance's split values, in the order of the class's shared names,
with defaults and constants copied in and the instance itself allowed
as a value. Nothing evaluates the body.

The stores' own sites prove the shortcut: each must have last stored,
for the class's current version, the instance's next position, which
also proves the store a plain __dict__ store (no descriptor or
__setattr__ intercepts it). The first constructions take the general
path and prove it; the class's instance plan then records the verdict,
and later constructions skip the plan lookup, the arity derivation and
the per-site checks. A class version change (a property or
__setattr__ added later, for instance) rebuilds the plan, so the stores
are proven again.

A two-attribute construction retires 17% fewer instructions.
---
 crates/weavepy-vm/src/lib.rs           | 304 +++++++++++++++++++++++++
 crates/weavepy-vm/src/types.rs         |   4 +
 tests/regrtest/test_store_only_init.py | 127 +++++++++++
 3 files changed, 435 insertions(+)
 create mode 100644 tests/regrtest/test_store_only_init.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 6593d48f..e6d93fb1 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -16520,6 +16520,48 @@ impl Interpreter {
             }
             return true;
         }
+        // A class whose construction its plan proved to be a store-only
+        // `__init__`'s stores (see `core_store_init`).
+        // SAFETY: as above.
+        let proven = unsafe { ty.instance_plan.peek() }
+            .and_then(Option::as_ref)
+            .filter(|(v, plan)| {
+                *v == ty.attr_version.get()
+                    && plan.store_init.load(std::sync::atomic::Ordering::Relaxed) == 1
+            })
+            .and_then(|(_, plan)| {
+                let (init, code) = plan.lean_init.as_ref()?;
+                // SAFETY: GIL-serialized raw read of the function's code
+                // cell; only the pointer is compared.
+                let same = std::ptr::eq(
+                    unsafe { Rc::as_ptr(&*init.code.as_ptr()) },
+                    Rc::as_ptr(code),
+                );
+                let missing = Self::missing_defaults(init, code, argc + 1).filter(|_| same)?;
+                Some((ty.clone(), init.clone(), code.clone(), missing))
+            });
+        if let Some((ty, init, code, missing)) = proven {
+            if self.core_store_init(
+                frame,
+                &ty,
+                &init,
+                &code,
+                None,
+                argc,
+                missing,
+                self_slot,
+                callee_slot,
+                sw.depth_cell,
+            ) {
+                frame.pc = pc as u32 + 1;
+                return true;
+            }
+        }
+        // SAFETY: as above.
+        let frame = unsafe { &mut *sw.cur };
+        let Object::Type(ty) = &frame.stack[callee_slot] else {
+            return false;
+        };
         let ty = ty.clone();
         let plan = self.instance_plan(&ty);
         // A named tuple: the tuple of the arguments is the whole call (see
@@ -16590,6 +16632,26 @@ impl Interpreter {
         let Some(missing) = Self::lean_init_missing(init, code, argc + 1) else {
             return false;
         };
+        // A store-only `__init__`: the stores themselves, as its sites
+        // last proved them.
+        if matches!(plan.native, crate::types::NativeKind::Plain)
+            && !plan.seeds_exception_args
+            && self.core_store_init(
+                frame,
+                &ty,
+                init,
+                code,
+                Some(&plan.store_init),
+                argc,
+                missing,
+                self_slot,
+                callee_slot,
+                sw.depth_cell,
+            )
+        {
+            frame.pc = pc as u32 + 1;
+            return true;
+        }
         // A leaf `__init__` (plain stores of its arguments into `self`)
         // runs frameless, as a leaf method call does (a plain instance
         // only).
@@ -16682,6 +16744,146 @@ impl Interpreter {
     /// unused instance was never seen).
     #[inline(never)]
     #[allow(clippy::too_many_arguments)]
+    /// [`Self::core_new`] for a store-only `__init__` (see
+    /// [`code_store_init`]): a fresh instance takes the arguments (or
+    /// constants) straight into its split values, in the order the
+    /// class's shared names hold them, when every store's site last
+    /// stored there for the class's current version (which proves the
+    /// store a plain `__dict__` one, see `core_store_new_attr`). The
+    /// arguments the stores don't take are released. `false` touches
+    /// nothing.
+    ///
+    /// `proving` is the class plan's verdict to record when the stores
+    /// prove (or disprove) the shape; `None` when the plan proved it.
+    #[allow(clippy::too_many_arguments)]
+    fn core_store_init(
+        &self,
+        frame: &mut Frame,
+        ty: &Rc<TypeObject>,
+        init: &Rc<crate::object::PyFunction>,
+        code: &Rc<CodeObject>,
+        proving: Option<&std::sync::atomic::AtomicU8>,
+        argc: usize,
+        missing: usize,
+        self_slot: usize,
+        callee_slot: usize,
+        depth_cell: *const std::cell::Cell<usize>,
+    ) -> bool {
+        let Some(shape) = code_store_init(code) else {
+            return false;
+        };
+        let Some(slots) = code_vm_ext(code).and_then(|e| e.field_slots.get()) else {
+            return false;
+        };
+        let ver = ty.attr_version.get();
+        if argc + 1 + missing != code.arg_count as usize
+            || ty.flags.is_builtin
+            || ty.native_kind.get() != 0
+            || ty.instances_need_finalize()
+            || crate::capi_watchers::dicts_active()
+            || crate::gil::free_threading_enabled()
+            // The ordinary call's `RecursionError` check.
+            // SAFETY: this thread's own depth cell.
+            || unsafe { (*depth_cell).get() } >= crate::recursion::recursion_limit()
+        {
+            return false;
+        }
+        if let Some(proving) = proving {
+            // Every store's site must have last stored the next position
+            // of this class version's instances. A site proven for another
+            // position never will be; one that hasn't stored for this
+            // version yet may still.
+            for (k, &(pc, _)) in shape.stores.iter().enumerate() {
+                match slots.get(pc as usize).map(FieldSlot::get) {
+                    Some((v, idx)) if v == ver && idx == k as u32 => {}
+                    Some((v, _)) if v == ver => {
+                        proving.store(2, std::sync::atomic::Ordering::Relaxed);
+                        return false;
+                    }
+                    _ => return false,
+                }
+            }
+        }
+        let Some(keys) = ty.shared_keys.get() else {
+            return false;
+        };
+        let consts = code_vm_ext(code).map_or(&[][..], |e| &e.objects[..]);
+        if keys.len() < shape.stores.len()
+            || shape
+                .stores
+                .iter()
+                .any(|&(_, src)| matches!(src, StoreSrc::Const(c) if c as usize >= consts.len()))
+        {
+            return false;
+        }
+        // Committed.
+        let (inst, _) = self.alloc_plain_instance_obj(ty);
+        let Object::Instance(i) = &inst else {
+            unreachable!("a plain class's instance")
+        };
+        // SAFETY: the instance is fresh and unshared.
+        if let Some(split) = unsafe { i.dict.split_cell().peek_mut() } {
+            split.reserve_for(|| ty.shared_keys.share(), keys.len());
+        }
+        // The parameters: the instance, the arguments (moved out of the
+        // stack on their last use), then the defaults.
+        let args = self_slot + 1;
+        let mut left = [0u8; 8];
+        left[..shape.uses.len()].copy_from_slice(&shape.uses);
+        let defaults = &init.defaults[init.defaults.len() - missing..];
+        for (k, &(_, src)) in shape.stores.iter().enumerate() {
+            let value = match src {
+                StoreSrc::Param(0) => inst.clone(),
+                StoreSrc::Param(j) => {
+                    let j = j as usize;
+                    left[j] -= 1;
+                    if j <= argc {
+                        let slot = &mut frame.stack[args + j - 1];
+                        if left[j] == 0 {
+                            std::mem::replace(slot, Object::Unbound)
+                        } else {
+                            slot.clone()
+                        }
+                    } else {
+                        defaults[j - argc - 1].clone()
+                    }
+                }
+                StoreSrc::Const(c) => consts[c as usize].clone(),
+                StoreSrc::Int(n) => Object::Int(n),
+            };
+            // SAFETY: a store with nothing else running; the positions are
+            // the instance's next ones (proved above).
+            if let Err(value) = unsafe { i.split_append(k, value) } {
+                // (Unreachable for a fresh instance over its class's
+                // names; the plain store keeps it correct regardless.)
+                let at = code.instructions[shape.stores[k].0 as usize].arg;
+                if let Some(name) = code_name_obj(code, at) {
+                    i.dict_shared()
+                        .borrow_mut()
+                        .insert(DictKey(name.clone()), value);
+                }
+            }
+        }
+        // The class, the self slot and the arguments the stores left.
+        // SAFETY: the operands leave the stack, each released once.
+        unsafe {
+            let base = frame.stack.as_ptr();
+            let n = frame.stack.len();
+            frame.stack.set_len(callee_slot);
+            for k in callee_slot..n {
+                let o = base.add(k).read();
+                if !matches!(o, Object::Unbound) {
+                    self.release(o);
+                }
+            }
+        }
+        frame.stack.push(inst);
+        if let Some(proving) = proving {
+            proving.store(1, std::sync::atomic::Ordering::Relaxed);
+        }
+        true
+    }
+
     fn core_leaf_init(
         &self,
         frame: &mut Frame,
@@ -53924,6 +54126,7 @@ impl Interpreter {
             lean_init,
             tuple_new,
             bare_alloc,
+            store_init: std::sync::atomic::AtomicU8::new(0),
         }
     }
 
@@ -70248,6 +70451,9 @@ struct CodeConstObjects {
     /// Whether every return is `return None` (see
     /// [`code_returns_only_none`]): `0` not yet decided, `1` no, `2` yes.
     returns_none: std::sync::atomic::AtomicU8,
+    /// The body's store-only `__init__` shape (see [`code_store_init`]),
+    /// or `None` when it isn't one.
+    store_init: std::sync::OnceLock<Option<Box<StoreInit>>>,
     /// The code's name and qualified name as interned strings: a fresh
     /// generator's `gi_name` and `gi_qualname` while its function's names
     /// are untouched.
@@ -72045,6 +72251,103 @@ fn code_returns_only_none(code: &CodeObject) -> bool {
     yes
 }
 
+/// Where a store-only `__init__`'s stored value comes from.
+#[derive(Clone, Copy)]
+enum StoreSrc {
+    /// The parameter at this position (`0` is the instance itself).
+    Param(u32),
+    /// This constant (an index into the code's constant objects).
+    Const(u32),
+    /// A small int literal.
+    Int(i64),
+}
+
+/// A *store-only* `__init__` (see [`code_store_init`]): its attribute
+/// stores into `self`, in order.
+struct StoreInit {
+    /// Each store's `STORE_ATTR` pc and the value it stores.
+    stores: Box<[(u32, StoreSrc)]>,
+    /// How many stores take each parameter (a parameter's last use moves
+    /// it, any earlier one copies it).
+    uses: Box<[u8]>,
+}
+
+/// The store-only shape of `code`: a body that is nothing but
+/// `self.name = <parameter or constant>` statements followed by
+/// `return None` (`RESUME` aside), as most `__init__`s are. Cached in the
+/// code's extension.
+fn code_store_init(code: &CodeObject) -> Option<&StoreInit> {
+    let ext = code_vm_ext(code)?;
+    ext.store_init
+        .get_or_init(|| store_init_shape(code).map(Box::new))
+        .as_deref()
+}
+
+fn store_init_shape(code: &CodeObject) -> Option<StoreInit> {
+    let nparams = code.arg_count as usize;
+    if nparams == 0
+        || nparams > 8
+        || code.is_generator
+        || code.is_coroutine
+        || code.is_async_generator
+        || !code.cellvars.is_empty()
+        || !code.freevars.is_empty()
+        || Interpreter::has_extended_params(code)
+    {
+        return None;
+    }
+    let ins = &code.instructions;
+    let mut stores = Vec::new();
+    let mut uses = vec![0u8; nparams];
+    let mut pc = 0;
+    while pc < ins.len() {
+        let i = ins[pc];
+        match i.op {
+            OpCode::Resume | OpCode::Nop | OpCode::NotTaken => pc += 1,
+            OpCode::LoadConst
+                if matches!(code.constants.get(i.arg as usize), Some(Constant::None))
+                    && ins.get(pc + 1).is_some_and(|n| n.op == OpCode::ReturnValue) =>
+            {
+                // The final `return None`: the shape holds when nothing
+                // else follows it.
+                if pc + 2 != ins.len() || stores.is_empty() || stores.len() > 16 {
+                    return None;
+                }
+                return Some(StoreInit {
+                    stores: stores.into_boxed_slice(),
+                    uses: uses.into_boxed_slice(),
+                });
+            }
+            _ => {
+                let src = match i.op {
+                    OpCode::LoadFast | OpCode::LoadFastBorrow | OpCode::LoadFastCheck
+                        if (i.arg as usize) < nparams =>
+                    {
+                        uses[i.arg as usize] = uses[i.arg as usize].checked_add(1)?;
+                        StoreSrc::Param(i.arg)
+                    }
+                    OpCode::LoadConst => StoreSrc::Const(i.arg),
+                    OpCode::LoadSmallInt => StoreSrc::Int(i64::from(i.arg)),
+                    _ => return None,
+                };
+                let recv = ins.get(pc + 1)?;
+                let store = ins.get(pc + 2)?;
+                if !matches!(
+                    recv.op,
+                    OpCode::LoadFast | OpCode::LoadFastBorrow | OpCode::LoadFastCheck
+                ) || recv.arg != 0
+                    || store.op != OpCode::StoreAttr
+                {
+                    return None;
+                }
+                stores.push((u32::try_from(pc + 2).ok()?, src));
+                pc += 3;
+            }
+        }
+    }
+    None
+}
+
 /// Whether `code` is an *effect leaf* (see `code_pure_leaf_decide`),
 /// deciding its leaf verdicts on first use.
 #[inline]
@@ -72244,6 +72547,7 @@ fn code_vm_ext_build(code: &CodeObject) -> &CodeConstObjects {
             leaf_sites: std::sync::OnceLock::new(),
             leaf_plan: std::sync::OnceLock::new(),
             returns_none: std::sync::atomic::AtomicU8::new(0),
+            store_init: std::sync::OnceLock::new(),
             gen_names: std::sync::OnceLock::new(),
             #[cfg(feature = "jit")]
             frame_jit: frame_jit::Slot::default(),
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index d98a4276..1a2d3ae2 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -863,6 +863,10 @@ pub struct InstancePlan {
     /// and not a module or generic-alias subclass (whose allocators
     /// differ).
     pub bare_alloc: bool,
+    /// Whether construction is the stores of a store-only `__init__` (see
+    /// `Interpreter::core_store_init`): `0` not yet proven, `1` proven
+    /// for this class version, `2` never.
+    pub store_init: std::sync::atomic::AtomicU8,
 }
 
 /// How a fresh instance's `native` payload is provisioned (see
diff --git a/tests/regrtest/test_store_only_init.py b/tests/regrtest/test_store_only_init.py
new file mode 100644
index 00000000..8625ce2e
--- /dev/null
+++ b/tests/regrtest/test_store_only_init.py
@@ -0,0 +1,127 @@
+# A class whose `__init__` only stores its parameters (or constants) into
+# `self` constructs by those stores alone once its store sites have
+# proven them plain `__dict__` stores. Every way the class or the function
+# can change under that shortcut must still be honored.
+
+import gc
+import weakref
+
+N = 300
+
+
+class V:
+    def __init__(self, x, y, z=7):
+        self.x = x
+        self.y = y
+        self.z = z
+        self.k = 5
+        self.label = "v"
+        self.me = self
+        self.again = x
+
+
+def build(n):
+    return [V(i, -i) for i in range(n)]
+
+
+for v in build(N)[-3:]:
+    assert v.me is v
+    assert v.z == 7 and v.k == 5 and v.label == "v" and v.again == v.x
+    assert list(vars(v)) == ["x", "y", "z", "k", "label", "me", "again"], vars(v)
+
+# A default given explicitly, and the argument stored twice.
+for i in range(N):
+    v = V([i], 2, z=None) if i % 2 else V([i], 2, None)
+    assert v.z is None and v.x is v.again and v.x == [i]
+
+# Arity errors still raise from the call.
+for args in [(), (1,), (1, 2, 3, 4)]:
+    try:
+        V(*args)
+    except TypeError as e:
+        assert "argument" in str(e), e
+    else:
+        raise AssertionError(args)
+
+# A property installed later on the class intercepts the store.
+seen = []
+
+
+def set_y(self, value):
+    seen.append(value)
+    self.__dict__["_y"] = value
+
+
+build(N)
+V.y = property(lambda self: self.__dict__["_y"], set_y)
+v = V(1, "yy")
+assert seen == ["yy"] and v.y == "yy" and "y" not in vars(v)
+del V.y
+for v in build(N)[-2:]:
+    assert "y" in vars(v)
+
+# A `__setattr__` added later sees every store.
+calls = []
+
+
+def tracing_setattr(self, name, value):
+    calls.append(name)
+    object.__setattr__(self, name, value)
+
+
+V.__setattr__ = tracing_setattr
+V(1, 2)
+assert calls == ["x", "y", "z", "k", "label", "me", "again"], calls
+del V.__setattr__
+calls.clear()
+build(N)
+assert calls == []
+
+
+# A subclass sharing the base's `__init__`, built in alternation.
+class W(V):
+    pass
+
+
+for i in range(N):
+    a, b = V(i, 0), W(i, 1)
+    assert type(a) is V and type(b) is W and b.me is b and b.y == 1
+
+# Replacing `__init__`'s code or its defaults takes effect at once.
+build(N)
+
+
+def other(self, x, y, z=7):
+    self.x = -x
+
+
+V.__init__.__code__ = other.__code__
+v = V(3, 4)
+assert vars(v) == {"x": -3}, vars(v)
+
+
+class D:
+    def __init__(self, a, b=1):
+        self.a = a
+        self.b = b
+
+
+for i in range(N):
+    assert D(i).b == 1
+D.__init__.__defaults__ = (99,)
+assert D(0).b == 99
+
+# Instances still die by reference count, and the class with its last.
+for _ in range(2):
+    class T:
+        def __init__(self, a):
+            self.a = a
+
+    for i in range(N):
+        T(i)
+    r = weakref.ref(T)
+    del T
+    gc.collect()
+    assert r() is None
+
+print("test_store_only_init: OK")

From 1720135816309350cf7fa23a1b583f507c320b8a Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 19:38:43 -0700
Subject: [PATCH 005/173] perf: read globals in line in lean compiled bodies

A body the frame JIT lowers lean (one whose IR comes out large) now
keeps the in-line LOAD_GLOBAL cache read: a few loads and compares
against the helper's full call. A LOAD_GLOBAL whose fused shape (len of
a local, a pure leaf call, a class constant) declines now fills the
in-line cache as any other read does, instead of calling the helper
every time. An exhausted string or bytes iterator also leaves the stack
natively, as a range's does.

Instructions retired: deepcopy -4.5%, oo_patterns -2.8%, exceptions
-2.4%, tomllib_loads -2.1%; suite geomean -0.7%.
---
 crates/weavepy-vm/src/frame_jit.rs | 59 ++++++++++++++++++------------
 1 file changed, 36 insertions(+), 23 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 6a500610..f49292e1 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1198,6 +1198,18 @@ unsafe extern "C" fn h_for_iter(st: *mut State, it: *mut Object, out: *mut Objec
             Some(v) => v,
             None => return 3,
         },
+        // An exhausted string's or bytes' cursor leaves the stack, as a
+        // range's does: releasing it runs no code.
+        PyIterator::Str { s, index } if unique && *index >= s.len() => {
+            // SAFETY: as above.
+            drop(unsafe { it.read() });
+            return 2;
+        }
+        PyIterator::Bytes { data, index } if unique && *index >= data.len() => {
+            // SAFETY: as above.
+            drop(unsafe { it.read() });
+            return 2;
+        }
         _ => return 3,
     };
     if let Object::Int(i) = v {
@@ -1824,27 +1836,28 @@ unsafe extern "C" fn h_load_global(
                     return fused(st, c, pc + 1);
                 }
             }
-            // A value no fused shape takes: the native code reads it in
-            // line while the dicts' stamps hold (a stamp changes with every
-            // mutation, so the value stays where it is).
-            _ => {
-                let c = &mut *cache;
-                // (A namespace handle's bits: one pointer-sized word.)
-                c.value = 0;
-                c.globals = std::ptr::addr_of!((*st.frame).globals).cast::<u64>().read();
-                c.gstamp = g_stamp;
-                c.gdata = gdict as u64;
-                (c.builtins, c.bstamp, c.bdata) = if builtin {
-                    let b = std::ptr::addr_of!((*st.frame).builtins)
-                        .cast::<u64>()
-                        .read();
-                    (b, b_stamp, bdict as u64)
-                } else {
-                    (0, 0, 0)
-                };
-                c.value = std::ptr::from_ref(v) as u64;
-            }
+            _ => {}
         }
+        // The value itself: the native code reads it in line while the
+        // dicts' stamps hold (a stamp changes with every mutation, so the
+        // value stays where it is). A fused shape that declined fills the
+        // cache too: the next read tries the shape again only through a
+        // miss.
+        let c = &mut *cache;
+        // (A namespace handle's bits: one pointer-sized word.)
+        c.value = 0;
+        c.globals = std::ptr::addr_of!((*st.frame).globals).cast::<u64>().read();
+        c.gstamp = g_stamp;
+        c.gdata = gdict as u64;
+        (c.builtins, c.bstamp, c.bdata) = if builtin {
+            let b = std::ptr::addr_of!((*st.frame).builtins)
+                .cast::<u64>()
+                .read();
+            (b, b_stamp, bdict as u64)
+        } else {
+            (0, 0, 0)
+        };
+        c.value = std::ptr::from_ref(v) as u64;
         dst.write(crate::clone_hot(v));
         PUSHED
     }
@@ -6849,9 +6862,9 @@ impl<'a> Lower<'a> {
         self.global_caches.push(cache);
         let helper = self.b.create_block();
         let done = self.b.create_block();
-        if self.cold {
-            self.b.ins().jump(helper, &[]);
-        } else {
+        // (In line even in a lean body: the cached read is a few loads
+        // and compares, the helper's a full call.)
+        {
             let ptr = self.ptr;
             let c = self.b.ins().iconst(ptr, cache_at);
             let frame = self.b.ins().load(ptr, FLAGS, self.st, S_FRAME);

From 74354b5d3915a8907ab4b96dc761770d8f1b18e7 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 18:30:13 -0700
Subject: [PATCH 006/173] perf: cheaper raises, deletes and with statements

Exceptions and context managers spent most of their time in bookkeeping
around the operation itself:

- `PUSH_EXC_INFO` kept a second copy of the handled exception per frame
  (`FrameExc::handlers`), which nothing read; `POP_EXCEPT` now releases
  the `exc_info_stack` entry it pops instead.
- An `AttributeError` with `name` and `obj`, and a generator's
  `StopIteration`, are built in one step (message, `args` and fields)
  instead of construction plus per-slot stores; the common pseudo-slot
  keys (now including `name` and `obj`) resolve by length and one
  comparison instead of interning.
- A `LOAD_ATTR` of a plain instance attribute that is certainly missing
  raises its `AttributeError` without the resolution passes.
- `del obj.attr` keeps an instance's split layout instead of
  materializing a dictionary: the last value just leaves, and any other
  deletion lays the rest out over the instance's own copy of the names
  (which class-keyed fast paths never match), as CPython's split
  dictionaries do.
- `LOAD_SPECIAL` runs natively in the frame JIT, and a `with` statement's
  enter and exit calls count as in-place method calls when its special
  sites found leaf or native methods, so with-heavy loops compile.

Per-iteration instructions (probes): with 1932 -> 1811, raise 3760 ->
3610, missing attribute caught 9989 -> 6394, del attr 1670 -> 1188,
contextmanager with 30037 -> 26789. Suite (instructions vs. the branch
base): context_managers -6.8%, exceptions -3.8%, raytrace -2.2%, geomean
0.9986.
---
 crates/weavepy-vm/src/builtin_types.rs      |  15 ++
 crates/weavepy-vm/src/error.rs              |  59 ++---
 crates/weavepy-vm/src/frame_jit.rs          |  52 +++-
 crates/weavepy-vm/src/inst_dict.rs          |  97 ++++++-
 crates/weavepy-vm/src/lib.rs                | 194 +++++++++-----
 crates/weavepy-vm/src/types.rs              |  23 +-
 tests/regrtest/test_exception_fast_paths.py | 271 ++++++++++++++++++++
 tests/regrtest/test_split_attr_delete.py    |  82 ++++++
 8 files changed, 693 insertions(+), 100 deletions(-)
 create mode 100644 tests/regrtest/test_exception_fast_paths.py
 create mode 100644 tests/regrtest/test_split_attr_delete.py

diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs
index dca1ca9d..fe2f2851 100644
--- a/crates/weavepy-vm/src/builtin_types.rs
+++ b/crates/weavepy-vm/src/builtin_types.rs
@@ -4875,6 +4875,17 @@ pub fn make_exception_with_class(class: Rc<TypeObject>, message: impl Into<Strin
 /// `message` attribute: `str()` and [`exception_message`] derive the
 /// text from `args` on demand, so nothing is formatted here.
 pub(crate) fn exception_from_parts(class: Rc<TypeObject>, arg: Option<Object>) -> Object {
+    exception_from_parts_with(class, arg, [])
+}
+
+/// [`exception_from_parts`] with the further pseudo-slots `extra`, in
+/// order, after the families' own (`AttributeError`'s `name` and `obj`):
+/// the instance's slots are built once, with no per-slot probe.
+pub(crate) fn exception_from_parts_with<const N: usize>(
+    class: Rc<TypeObject>,
+    arg: Option<Object>,
+    extra: [(&'static str, Object); N],
+) -> Object {
     use crate::types::{slot_key, PyInstance, SlotStorage};
     let families = crate::exc_family_flags(&class);
     let inst = PyInstance::new(class);
@@ -4917,6 +4928,10 @@ pub(crate) fn exception_from_parts(class: Rc<TypeObject>, arg: Option<Object>) -
         inst.note_slot_store(&msg);
         entries.push((slot_key("msg"), msg));
     }
+    for (name, value) in extra {
+        inst.note_slot_store(&value);
+        entries.push((slot_key(name), value));
+    }
     *inst.slots.borrow_mut() = SlotStorage::from_entries(entries);
     Object::Instance(Rc::new(inst))
 }
diff --git a/crates/weavepy-vm/src/error.rs b/crates/weavepy-vm/src/error.rs
index fa1aabb9..3b2e1664 100644
--- a/crates/weavepy-vm/src/error.rs
+++ b/crates/weavepy-vm/src/error.rs
@@ -293,18 +293,24 @@ pub fn attribute_error(message: impl Into<String>) -> RuntimeError {
 /// CPython's C raise sites populate (PEP 3134-adjacent; used by
 /// suggestion machinery and asserted on by `test_exceptions`).
 pub fn attribute_error_named(obj: &Object, name: &str) -> RuntimeError {
-    let err = attribute_error(format!(
-        "'{}' object has no attribute '{}'",
-        obj.type_name_owned(),
-        name
-    ));
-    if let RuntimeError::PyException(pe) = &err {
-        if let Object::Instance(inst) = &pe.instance {
-            inst.slot_set("name", Object::from_str(name));
-            inst.slot_set("obj", obj.clone());
-        }
-    }
-    err
+    // Built in one step (message, `args` and both fields): a failing
+    // lookup under `getattr(o, n, default)` or `except AttributeError`
+    // is a hot path.
+    let type_name = obj.type_name_owned();
+    let mut message = String::with_capacity(type_name.len() + name.len() + 30);
+    message.push('\'');
+    message.push_str(&type_name);
+    message.push_str("' object has no attribute '");
+    message.push_str(name);
+    message.push('\'');
+    let instance = crate::builtin_types::exception_from_parts_with(
+        crate::builtin_types::builtin_types()
+            .attribute_error
+            .clone(),
+        Some(Object::from_str(message)),
+        [("name", Object::from_str(name)), ("obj", obj.clone())],
+    );
+    RuntimeError::PyException(PyException::new(instance))
 }
 
 pub fn key_error(message: impl Into<String>) -> RuntimeError {
@@ -420,23 +426,18 @@ pub fn stop_async_iteration() -> RuntimeError {
 /// of a `return` statement. The wrapped value is exposed as `.value`
 /// on the exception instance.
 pub fn stop_iteration_with(value: Object) -> RuntimeError {
-    let pe = PyException::new(crate::builtin_types::make_exception_with_class(
-        crate::builtin_types::builtin_types().stop_iteration.clone(),
-        "",
-    ));
-    if let Object::Instance(ref inst) = pe.instance {
-        inst.slot_set("value", value.clone());
-        // A bare `return` (value None) raises `StopIteration()` with
-        // *empty* args, so `str(e)` renders bare and `e.args` is `()` —
-        // CPython's `gen_return` only packs non-None return values.
-        let args = if matches!(value, Object::None) {
-            Object::new_tuple(Vec::new())
-        } else {
-            Object::new_tuple_array([value])
-        };
-        inst.slot_set("args", args);
-    }
-    RuntimeError::PyException(pe)
+    // `value` is the return value; a bare `return` (value None) raises
+    // `StopIteration()` with *empty* args, so `str(e)` renders bare and
+    // `e.args` is `()` — CPython's `gen_return` only packs non-None
+    // return values. (The family's construction sets `.value` from the
+    // argument, `None` without one.)
+    let arg = (!matches!(value, Object::None)).then_some(value);
+    RuntimeError::PyException(PyException::new(
+        crate::builtin_types::exception_from_parts(
+            crate::builtin_types::builtin_types().stop_iteration.clone(),
+            arg,
+        ),
+    ))
 }
 
 pub fn not_implemented_error(message: impl Into<String>) -> RuntimeError {
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index f49292e1..6b208522 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -417,6 +417,49 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
             }
         }
     }
+    // So do a `with` statement's `__enter__` and `__exit__` calls when
+    // its special-method sites found methods that run in place: the
+    // enter call follows its `LOAD_SPECIAL`, and a normal exit's
+    // `CALL 3` (over three `None`s) pairs with the innermost open exit
+    // site.
+    {
+        use weavepy_compiler::bytecode::{SPECIAL_ENTER, SPECIAL_EXIT};
+        let mut open = Vec::new();
+        for pc in 0..ins.len() {
+            let i = ins[pc];
+            match i.op {
+                OpCode::LoadSpecial if i.arg == SPECIAL_EXIT => open.push(pc),
+                OpCode::LoadSpecial if i.arg == SPECIAL_ENTER => {
+                    if ins
+                        .get(pc + 1)
+                        .is_some_and(|c| c.op == OpCode::Call && c.arg == 0)
+                        && crate::method_site_in_place(ext, pc)
+                    {
+                        helped[pc + 1] = true;
+                    }
+                }
+                OpCode::Call
+                    if i.arg == 3
+                        && pc >= 3
+                        && ins[pc - 3..pc].iter().all(|c| {
+                            c.op == OpCode::LoadConst
+                                && matches!(
+                                    code.constants.get(c.arg as usize),
+                                    Some(weavepy_compiler::Constant::None)
+                                )
+                        }) =>
+                {
+                    if open
+                        .pop()
+                        .is_some_and(|x| crate::method_site_in_place(ext, x))
+                    {
+                        helped[pc] = true;
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
     let t = tuning();
     // What running an instruction natively saves, in halves of an
     // in-line instruction's: one run in line saves its whole dispatch and
@@ -461,6 +504,7 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
             | OpCode::ListToTuple
             | OpCode::UnpackEx
             | OpCode::LoadSuperAttr
+            | OpCode::LoadSpecial
             | OpCode::PushExcInfo
             | OpCode::CheckExcMatch
             | OpCode::PopExcept
@@ -1270,13 +1314,17 @@ unsafe extern "C" fn h_container(
             return Interpreter::core_super_attr(&*code, ins, pc as usize, st.stack, len)
                 .map_or(u64::MAX, |n| n as u64);
         }
+        if ins.op == OpCode::LoadSpecial {
+            return Interpreter::core_load_special(&*code, ins, pc as usize, st.stack, len, st.cap)
+                .map_or(u64::MAX, |n| n as u64);
+        }
         if matches!(
             ins.op,
             OpCode::PushExcInfo | OpCode::CheckExcMatch | OpCode::PopExcept
         ) {
             let interp = &mut *st.interp.cast_mut();
             return interp
-                .core_exc_op(&mut (*st.frame).exc, ins, st.stack, len, st.cap)
+                .core_exc_op(ins, st.stack, len, st.cap)
                 .map_or(u64::MAX, |n| n as u64);
         }
         if let Some(n) = Interpreter::core_container_op(ins, st.stack, len, st.cap) {
@@ -4028,6 +4076,7 @@ fn native_op(op: OpCode) -> bool {
             | OpCode::ListToTuple
             | OpCode::UnpackEx
             | OpCode::LoadSuperAttr
+            | OpCode::LoadSpecial
             | OpCode::PushExcInfo
             | OpCode::CheckExcMatch
             | OpCode::PopExcept
@@ -4995,6 +5044,7 @@ impl<'a> Lower<'a> {
             | OpCode::UnpackSequence
             | OpCode::UnpackEx
             | OpCode::LoadSuperAttr
+            | OpCode::LoadSpecial
             | OpCode::PushExcInfo
             | OpCode::CheckExcMatch
             | OpCode::PopExcept
diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index b8c2b35b..ca9f8872 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -118,15 +118,49 @@ impl SharedKeys {
     /// Publish the `str` name `name` as the next name; `None` when the
     /// table is full. The caller holds the GIL outside free-threaded mode.
     fn push(&self, name: &SharedStr) -> Option<usize> {
+        self.push_key(
+            DictKey(Object::Str(name.clone())),
+            crate::object::py_str_hash(name),
+        )
+    }
+
+    /// Remove the `i`th name, closing up the rest: only for a table no
+    /// one else holds (an instance's private names, see
+    /// [`SplitValues::remove_str`]).
+    fn remove_at(&mut self, i: usize) {
+        let n = *self.len.get_mut();
+        if i >= n {
+            return;
+        }
+        // SAFETY: `&mut self` is exclusive; the first `n` names are
+        // initialized, the `i`th drops once, and the later ones move down
+        // one slot each.
+        unsafe {
+            self.keys[i].get_mut().assume_init_drop();
+            for k in i + 1..n {
+                let key = self.keys[k].get_mut().assume_init_read();
+                self.keys[k - 1].get_mut().write(key);
+                *self.hashes[k - 1].get_mut() = *self.hashes[k].get_mut();
+            }
+        }
+        *self.len.get_mut() = n - 1;
+        let mut filter = 0u64;
+        for h in &mut self.hashes[..n - 1] {
+            filter |= 1 << (*h.get_mut() & 63);
+        }
+        *self.filter.get_mut() = filter;
+    }
+
+    /// [`Self::push`] of a name key whose Python hash is `hash`.
+    fn push_key(&self, key: DictKey, hash: i64) -> Option<usize> {
         let n = self.len.load(Ordering::Relaxed);
         if n >= SHARED_KEYS_CAP {
             return None;
         }
-        let hash = crate::object::py_str_hash(name);
         // SAFETY: slot `n` is unpublished, so no reader can see it, and
         // the GIL serializes writers.
         unsafe {
-            (*self.keys[n].get()).write(DictKey(Object::Str(name.clone())));
+            (*self.keys[n].get()).write(key);
             *self.hashes[n].get() = hash;
         }
         self.filter.fetch_or(1 << (hash & 63), Ordering::Relaxed);
@@ -639,6 +673,47 @@ impl SplitValues {
         Ok(None)
     }
 
+    /// Remove attribute `name`, returning its value (`None` when it is not
+    /// set). The last value simply leaves (the instance then holds one
+    /// name fewer of the same table); removing any other lays the rest out
+    /// over a private copy of their names, in order. Compiled code checks
+    /// an instance's names against its class's before using a position, so
+    /// such an instance takes the general paths from then on, as one with
+    /// a real dictionary does, without materializing one (CPython's split
+    /// dictionaries also keep their layout across a deletion).
+    pub fn remove_str(&mut self, name: &str) -> Option<Object> {
+        let i = self.position_str(name)?;
+        let h = self.block?.as_ptr();
+        // SAFETY: the block is live while owned, `i` is below its length,
+        // and every value from `i` on moves down one slot exactly once
+        // (the removed one moves out first).
+        unsafe {
+            let n = (*h).len as usize;
+            let values = Self::values_ptr(h);
+            let removed = values.add(i).read();
+            if i + 1 < n {
+                let mut keys = Rc::from_raw((*h).keys);
+                match Rc::get_mut(&mut keys) {
+                    // Names already private to this instance (an earlier
+                    // deletion's) close up in place.
+                    Some(own) => own.remove_at(i),
+                    None => {
+                        let fresh = SharedKeys::default();
+                        for k in (0..n).filter(|&k| k != i) {
+                            let key = keys.get(k).expect("a set value's name").clone();
+                            fresh.push_key(key, *keys.hashes[k].get());
+                        }
+                        keys = Rc::new(fresh);
+                    }
+                }
+                (*h).keys = Rc::into_raw(keys);
+                std::ptr::copy(values.add(i + 1), values.add(i), n - i - 1);
+            }
+            (*h).len = n as u32 - 1;
+            Some(removed)
+        }
+    }
+
     /// Move every value out (the caller drops them after releasing the
     /// cell) and forget the names.
     pub fn take(&mut self) -> Vec<Object> {
@@ -1258,6 +1333,24 @@ impl crate::types::PyInstance {
         }
     }
 
+    /// Delete attribute `name` through the split layout (see
+    /// [`SplitValues::remove_str`]): `Some` with the removed value, or with
+    /// `None` when the instance has no such attribute; `None` when the
+    /// instance has (or needs) a real dictionary. Runs no code: the caller
+    /// releases the value. The caller has established that a plain
+    /// `__dict__` deletion is what this `del` means.
+    pub fn split_remove(&self, name: &str) -> Option<Option<Object>> {
+        if self.dict.published().is_some()
+            || self.c_body.get() != 0
+            || crate::gil::free_threading_enabled()
+            || self.cls_raw().native_kind.get() != 0
+        {
+            return None;
+        }
+        let mut split = self.dict.split_cell().try_borrow_mut().ok()?;
+        Some(split.remove_str(name))
+    }
+
     /// Store `value` under the interned name `name` through the split
     /// layout: the displaced value on an overwrite, or `Err` handing the
     /// value back when the instance needs (or has) a real dictionary.
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index e6d93fb1..863f2b54 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -262,12 +262,6 @@ struct FrameRare {
 /// handled an exception carries.
 #[derive(Default)]
 struct FrameExc {
-    /// Stack of currently-handled exceptions. `PUSH_EXC_INFO` pushes
-    /// onto this; `POP_EXCEPT` pops; `RERAISE 1` re-raises the top.
-    /// Each entry is tagged with the pc just past its handler body
-    /// (the `PUSH_EXC_INFO` arg) so the unwinder can discard handlers an
-    /// exception propagates *out of* (see `handle_exception`).
-    handlers: Vec<(u32, PyException)>,
     /// A *generator/coroutine* frame's own handled-exception entries,
     /// detached from the interpreter-wide `exc_info_stack` while it is
     /// suspended at a `yield`. CPython stores this as the generator's
@@ -325,12 +319,6 @@ impl Frame {
         self.exc.get_or_insert_with(Box::default)
     }
 
-    /// Pop the innermost [`FrameExc::handlers`] entry.
-    #[inline]
-    fn exc_handlers_pop(&mut self) -> Option<(u32, PyException)> {
-        self.exc.as_mut()?.handlers.pop()
-    }
-
     /// Whether the frame holds detached [`FrameExc::saved`] entries.
     #[inline]
     fn has_saved_exc_info(&self) -> bool {
@@ -5712,7 +5700,6 @@ impl Interpreter {
             frame.stack.pop();
         }
         for _ in 0..jump.exc_pops {
-            frame.exc_handlers_pop();
             self.exc_info_stack.borrow_mut().pop();
         }
         frame.pc = jump.target_pc;
@@ -8164,7 +8151,6 @@ impl Interpreter {
         // activation never left the quiet loop's leaf paths).
         if !act.clean {
             if let Some(exc) = fr.exc.as_mut() {
-                exc.handlers.clear();
                 exc.saved.clear();
                 exc.cleanup_lasti = None;
                 exc.pending_lasti = None;
@@ -10921,6 +10907,75 @@ impl Interpreter {
         CoreAttr::Done
     }
 
+    /// [`Self::leaf_load_special`] over the `len` stack slots at `base`
+    /// (room to `cap`), for compiled code: `LOAD_SPECIAL` (`ins`, at `pc`
+    /// of `code`) of an instance whose class resolves the method to a
+    /// plain function or a native class's method, from the site's slot or
+    /// filling it. Returns the new stack length, or `None`, having touched
+    /// nothing.
+    ///
+    /// # Safety
+    ///
+    /// The `len` slots at `base` are initialized operand stack entries.
+    pub(crate) unsafe fn core_load_special(
+        code: &CodeObject,
+        ins: weavepy_compiler::Instruction,
+        pc: usize,
+        base: *mut Object,
+        len: usize,
+        cap: usize,
+    ) -> Option<usize> {
+        use weavepy_compiler::bytecode::{SPECIAL_ENTER, SPECIAL_EXIT};
+        if len == 0 || len >= cap {
+            return None;
+        }
+        // SAFETY: `len > 0`.
+        let top = unsafe { base.add(len - 1) };
+        let Object::Instance(inst) = (unsafe { &*top }) else {
+            return None;
+        };
+        let cls = inst.cls_raw();
+        let ver = cls.attr_version.get();
+        let ms = code_method_slot(code, pc as u32);
+        let held = ms.and_then(|ms| {
+            ms.get_held(ver)
+                .map(Object::Function)
+                .or_else(|| ms.get_inst_builtin(ver).map(Object::Builtin))
+        });
+        let method = match held {
+            Some(m) => m,
+            None => {
+                let name = match ins.arg {
+                    SPECIAL_ENTER => "__enter__",
+                    SPECIAL_EXIT => "__exit__",
+                    _ => return None,
+                };
+                match cls.lookup(name) {
+                    Some(Object::Function(f)) => {
+                        if let Some(ms) = ms {
+                            ms.set(ver, &f);
+                        }
+                        Object::Function(f)
+                    }
+                    Some(Object::Builtin(b)) if b.binds_instance && native_call_ic_safe(b.name) => {
+                        if let Some(ms) = ms {
+                            ms.set_inst_builtin(ver, &b);
+                        }
+                        Object::Builtin(b)
+                    }
+                    _ => return None,
+                }
+            }
+        };
+        // SAFETY: `len < cap`; the instance moves up into the self slot.
+        unsafe {
+            let recv = top.read();
+            top.write(method);
+            base.add(len).write(recv);
+        }
+        Some(len + 1)
+    }
+
     /// The core loop's `LOAD_FAST x` (at `pc`, of the heap local `other`)
     /// followed by `x.m(<simple arguments>)`: a native method the site
     /// cached for the receiver's class, a leaf method the site verified,
@@ -14517,7 +14572,7 @@ impl Interpreter {
                     OpCode::PushExcInfo | OpCode::CheckExcMatch | OpCode::PopExcept => {
                         // SAFETY: the `len` slots at `base` are the core
                         // loop's initialized stack, with room to `cap`.
-                        match unsafe { self.core_exc_op(&mut frame.exc, ins, base, len, cap) } {
+                        match unsafe { self.core_exc_op(ins, base, len, cap) } {
                             Some(n) => len = n,
                             None => break None,
                         }
@@ -17044,9 +17099,8 @@ impl Interpreter {
     }
 
     /// An `except` clause's `PUSH_EXC_INFO`, `CHECK_EXC_MATCH` or
-    /// `POP_EXCEPT` (`ins`) of a frame (its handler state `exc`) over the
-    /// `len` stack slots at `base`
-    /// (room to `cap`), as the full handlers run their common shapes: an
+    /// `POP_EXCEPT` (`ins`) over the `len` stack slots at `base` (room to
+    /// `cap`), as the full handlers run their common shapes: an
     /// exception instance on entry, class operands that are
     /// `BaseException` subclasses. Returns the new stack length, or
     /// `None`, having touched nothing. A `POP_EXCEPT`'s releases may queue
@@ -17059,7 +17113,6 @@ impl Interpreter {
     #[inline(never)]
     unsafe fn core_exc_op(
         &mut self,
-        exc: &mut Option<Box<FrameExc>>,
         ins: weavepy_compiler::Instruction,
         base: *mut Object,
         len: usize,
@@ -17088,11 +17141,9 @@ impl Interpreter {
                     base.add(len - 1).write(prev);
                     base.add(len).write(top);
                 }
-                let pe = PyException::new(instance);
-                exc.get_or_insert_with(Box::default)
-                    .handlers
-                    .push((ins.arg, pe.clone()));
-                self.exc_info_stack.borrow_mut().push(pe);
+                self.exc_info_stack
+                    .borrow_mut()
+                    .push(PyException::new(instance));
                 Some(len + 1)
             }
             OpCode::CheckExcMatch => {
@@ -17116,13 +17167,11 @@ impl Interpreter {
                 }
                 // SAFETY: the slot at `len - 1` is initialized.
                 let prev = unsafe { base.add(len - 1).read() };
-                let popped = exc.as_mut().and_then(|e| e.handlers.pop());
                 let top = self.exc_info_stack.borrow_mut().pop();
                 // See the full handler: the handled exception dies here
                 // unless something else holds it.
                 self.recheck_frame_observed = true;
-                drop(top);
-                if let Some((_, pe)) = popped {
+                if let Some(pe) = top {
                     self.release(pe.instance);
                 }
                 self.release(prev);
@@ -22152,7 +22201,6 @@ impl Interpreter {
             globals,
             builtins,
             rare,
-            exc: frame_exc,
             ..
         } = frame;
         let code: &CodeObject = code_rc;
@@ -22933,10 +22981,15 @@ impl Interpreter {
                     let Some(Object::Str(key)) = code_name_obj(code, ins.arg) else {
                         break;
                     };
-                    let removed = inst
-                        .dict_cell()
-                        .borrow_mut()
-                        .shift_remove(&DictKey(Object::Str(key.clone())));
+                    // (The split layout deletes in place; see
+                    // `PyInstance::split_remove`.)
+                    let removed = match inst.split_remove(key) {
+                        Some(removed) => removed,
+                        None => inst
+                            .dict_cell()
+                            .borrow_mut()
+                            .shift_remove(&DictKey(Object::Str(key.clone()))),
+                    };
                     // A missing attribute raises through the full path.
                     let Some(removed) = removed else {
                         break;
@@ -23955,12 +24008,7 @@ impl Interpreter {
                         .map_or(Object::None, |pe| pe.instance.clone());
                     stack.push(prev);
                     stack.push(exc.clone());
-                    let pe = PyException::new(exc);
-                    frame_exc
-                        .get_or_insert_with(Box::default)
-                        .handlers
-                        .push((ins.arg, pe.clone()));
-                    self.exc_info_stack.borrow_mut().push(pe);
+                    self.exc_info_stack.borrow_mut().push(PyException::new(exc));
                     last = pc;
                     pc += 1;
                 }
@@ -23979,15 +24027,13 @@ impl Interpreter {
                 }
                 OpCode::PopExcept => {
                     let Some(prev) = stack.pop() else { break };
-                    let popped = frame_exc.as_mut().and_then(|e| e.handlers.pop());
                     let top = self.exc_info_stack.borrow_mut().pop();
                     // See the full handler: the handled exception dies
                     // here unless something else holds it.
                     self.recheck_frame_observed = true;
                     last = pc;
                     pc += 1;
-                    drop(top);
-                    if let Some((_, pe)) = popped {
+                    if let Some(pe) = top {
                         self.release(pe.instance);
                     }
                     self.release(prev);
@@ -30089,10 +30135,7 @@ impl Interpreter {
                 if !matches!(matched, Object::None) {
                     let pe = PyException::new(matched.clone());
                     if let Some(top) = self.exc_info_stack.borrow_mut().last_mut() {
-                        *top = pe.clone();
-                    }
-                    if let Some(top) = frame.exc.as_mut().and_then(|e| e.handlers.last_mut()) {
-                        top.1 = pe;
+                        *top = pe;
                     }
                 }
                 frame.push(rest);
@@ -30115,17 +30158,7 @@ impl Interpreter {
                 frame.push(prev);
                 frame.push(exc.clone());
                 if let Object::Instance(_) = &exc {
-                    let pe = PyException::new(exc);
-                    // `ins.arg` is the pc just past this handler body
-                    // (back-patched by the compiler). Tag the entry so
-                    // the unwinder can drop it when an exception escapes
-                    // the handler to an enclosing `try`. A 0 arg means
-                    // the code carries no tag (e.g. older marshalled
-                    // bytecode); leave it untagged and let frame-exit
-                    // reconciliation handle any residue.
-                    let body_end = ins.arg;
-                    frame.exc_mut().handlers.push((body_end, pe.clone()));
-                    self.exc_info_stack.borrow_mut().push(pe);
+                    self.exc_info_stack.borrow_mut().push(PyException::new(exc));
                 }
             }
             OpCode::PopExcept => {
@@ -30135,14 +30168,13 @@ impl Interpreter {
                 // `sys.exc_info()` restore, so the value itself is
                 // discarded).
                 let _prev = frame.pop()?;
-                let popped = frame.exc_handlers_pop();
-                drop(self.exc_info_stack.borrow_mut().pop());
+                let popped = self.exc_info_stack.borrow_mut().pop();
                 // CPython clears the just-handled exception at the end of an
                 // `except` block (the implicit `del` of the bound name plus
                 // the per-frame exc-state pop): it dies here unless something
                 // else holds it.
                 self.recheck_frame_observed = true;
-                if let Some((_, pe)) = popped {
+                if let Some(pe) = popped {
                     self.release(pe.instance);
                 }
             }
@@ -45689,14 +45721,41 @@ impl Interpreter {
         }
         // SAFETY: as above (both declined untouched).
         let frame = unsafe { &mut *sw.cur };
-        let arg = frame.code.instructions[pc].arg;
+        let ins = frame.code.instructions[pc];
         frame.pc = pc as u32 + 1;
         let pending = self.core_pending_enter(sw, frame, pc);
-        let r = self.load_attr_step(frame, pc as u32, arg);
+        let r = match ins.op {
+            OpCode::LoadAttr => match self.load_attr_certain_miss(frame, ins.arg) {
+                Some(err) => Err(err),
+                None => self.load_attr_step(frame, pc as u32, ins.arg),
+            },
+            _ => self.load_attr_step(frame, pc as u32, ins.arg),
+        };
         self.core_pending_exit(sw, pending);
         r
     }
 
+    /// A `LOAD_ATTR` (name `name_idx`) of a plain instance that has no
+    /// such attribute anywhere ([`attr_certainly_missing`]): the receiver
+    /// leaves the stack, released as the full load releases it, and the
+    /// `AttributeError` the default lookup ends with comes back without
+    /// its resolution passes. `None` touches nothing.
+    #[cfg(feature = "jit")]
+    #[inline(never)]
+    fn load_attr_certain_miss(&mut self, frame: &mut Frame, name_idx: u32) -> Option<RuntimeError> {
+        let recv @ Object::Instance(_) = frame.stack.last()? else {
+            return None;
+        };
+        let name = frame.code.names.get(name_idx as usize)?;
+        if !attr_certainly_missing(recv, name) {
+            return None;
+        }
+        let recv = frame.stack.pop()?;
+        let err = crate::error::attribute_error_named(&recv, name);
+        self.release(recv);
+        Some(err)
+    }
+
     fn specialized_load_attr(
         &mut self,
         frame: &mut Frame,
@@ -48547,10 +48606,15 @@ impl Interpreter {
                 _ => {}
             }
         }
-        let removed = inst
-            .dict_cell()
-            .borrow_mut()
-            .shift_remove(&DictKey(Object::from_str(name)));
+        // The split layout deletes in place (see `PyInstance::split_remove`);
+        // a real dictionary is watchable.
+        let removed = match inst.split_remove(name) {
+            Some(removed) => removed,
+            None => inst
+                .dict_cell()
+                .borrow_mut()
+                .shift_remove(&DictKey(Object::from_str(name))),
+        };
         // Watched instance `__dict__` observes attribute deletion as
         // PyDict_EVENT_DELETED (test_watchers test_object_dict).
         if removed.is_some() && crate::capi_watchers::dicts_active() {
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 1a2d3ae2..bad33144 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -2216,7 +2216,7 @@ fn slot_name_eq(stored: &str, name: &str) -> bool {
 /// populates (`args`, `__traceback__`, the chaining links) share one
 /// interned key each instead of allocating a string per exception.
 pub(crate) fn slot_key(name: &str) -> DictKey {
-    const COMMON: [&str; 8] = [
+    const COMMON: [&str; 10] = [
         "args",
         "__traceback__",
         "__context__",
@@ -2225,9 +2225,26 @@ pub(crate) fn slot_key(name: &str) -> DictKey {
         "message",
         "value",
         "msg",
+        "name",
+        "obj",
     ];
-    static KEYS: std::sync::OnceLock<[Object; 8]> = std::sync::OnceLock::new();
-    if let Some(i) = COMMON.iter().position(|c| *c == name) {
+    static KEYS: std::sync::OnceLock<[Object; 10]> = std::sync::OnceLock::new();
+    // (By length first: one comparison settles each common name.)
+    let common = match name.len() {
+        3 if name == "msg" => Some(7),
+        3 if name == "obj" => Some(9),
+        4 if name == "args" => Some(0),
+        4 if name == "name" => Some(8),
+        5 if name == "value" => Some(6),
+        7 if name == "message" => Some(5),
+        9 if name == "__cause__" => Some(3),
+        11 if name == "__context__" => Some(2),
+        13 if name == "__traceback__" => Some(1),
+        20 if name == "__suppress_context__" => Some(4),
+        _ => None,
+    };
+    if let Some(i) = common {
+        debug_assert_eq!(COMMON[i], name);
         let keys = KEYS.get_or_init(|| COMMON.map(crate::stdlib::sys::intern_name));
         return DictKey(keys[i].clone());
     }
diff --git a/tests/regrtest/test_exception_fast_paths.py b/tests/regrtest/test_exception_fast_paths.py
new file mode 100644
index 00000000..898ac4c0
--- /dev/null
+++ b/tests/regrtest/test_exception_fast_paths.py
@@ -0,0 +1,271 @@
+"""The raise/catch fast paths, the one-step AttributeError and
+StopIteration constructions, and compiled `with` statements keep CPython's
+observable behavior: handled-exception nesting, chaining, exception fields,
+special-method lookup and its invalidation."""
+
+import sys
+import threading
+import contextlib
+
+
+# --- Handled-exception nesting (sys.exc_info, bare raise, context) -------
+
+def nested_handlers():
+    out = []
+    for i in range(300):
+        try:
+            raise KeyError(i)
+        except KeyError as outer:
+            assert sys.exc_info()[1] is outer
+            try:
+                raise ValueError(i)
+            except ValueError as inner:
+                assert sys.exc_info()[1] is inner
+                assert inner.__context__ is outer
+            assert sys.exc_info()[1] is outer
+            try:
+                raise
+            except KeyError as again:
+                assert again is outer
+        assert sys.exc_info() == (None, None, None)
+        out.append(i)
+    return len(out)
+
+
+assert nested_handlers() == 300
+
+
+def reraiser():
+    try:
+        {}["k"]
+    except KeyError:
+        raise
+
+
+for _ in range(200):
+    try:
+        reraiser()
+    except KeyError as e:
+        assert e.args == ("k",)
+        tb = e.__traceback__
+        names = []
+        while tb is not None:
+            names.append(tb.tb_frame.f_code.co_name)
+            tb = tb.tb_next
+        assert names == ["<module>", "reraiser"], names
+
+
+# The handled exception dies at the end of its handler.
+class Fin(Exception):
+    dead = 0
+
+    def __del__(self):
+        Fin.dead += 1
+
+
+for i in range(100):
+    try:
+        raise Fin()
+    except Fin:
+        pass
+    assert Fin.dead == i + 1, (i, Fin.dead)
+
+
+# except* still splits and rebinds the handled exception.
+try:
+    raise ExceptionGroup("g", [ValueError(1), KeyError(2)])
+except* ValueError as eg:
+    assert isinstance(sys.exc_info()[1], ExceptionGroup)
+    assert [type(x) for x in eg.exceptions] == [ValueError]
+except* KeyError as eg:
+    assert [type(x) for x in eg.exceptions] == [KeyError]
+
+
+# --- AttributeError ------------------------------------------------------
+
+class Plain:
+    cls_attr = 1
+
+
+class WithGetattr:
+    def __getattr__(self, name):
+        return name.upper()
+
+
+class Slotted:
+    __slots__ = ("x",)
+
+
+p = Plain()
+for _ in range(300):
+    try:
+        p.missing
+    except AttributeError as e:
+        assert e.name == "missing" and e.obj is p
+        assert e.args == ("'Plain' object has no attribute 'missing'",), e.args
+        assert str(e) == "'Plain' object has no attribute 'missing'"
+    else:
+        raise AssertionError("no AttributeError")
+    assert getattr(p, "missing", 7) == 7
+    assert not hasattr(p, "missing")
+    assert WithGetattr().abc == "ABC"
+    s = Slotted()
+    try:
+        s.x
+    except AttributeError as e:
+        assert e.name == "x"
+    try:
+        s.y
+    except AttributeError as e:
+        assert e.name == "y" and e.obj is s
+p.missing = 5
+assert p.missing == 5
+del p.missing
+try:
+    p.missing
+except AttributeError as e:
+    assert e.obj is p
+
+
+# --- StopIteration from a generator's return -----------------------------
+
+def gen_ret(v):
+    yield 1
+    return v
+
+
+for v in (None, 0, "x", (1, 2)):
+    g = gen_ret(v)
+    next(g)
+    try:
+        next(g)
+    except StopIteration as e:
+        assert e.value == v
+        assert e.args == (() if v is None else (v,)), e.args
+    else:
+        raise AssertionError("no StopIteration")
+
+
+# --- `with` statements ---------------------------------------------------
+
+class CM:
+    def __init__(self):
+        self.log = []
+
+    def __enter__(self):
+        self.log.append("enter")
+        return self
+
+    def __exit__(self, t, v, tb):
+        self.log.append(("exit", t))
+        return False
+
+
+class Suppress:
+    def __enter__(self):
+        return 1
+
+    def __exit__(self, t, v, tb):
+        return t is not None and issubclass(t, KeyError)
+
+
+def run_with(n):
+    cm = CM()
+    sup = Suppress()
+    lock = threading.Lock()
+    rlock = threading.RLock()
+    total = 0
+    for i in range(n):
+        with cm as c:
+            assert c is cm
+            total += 1
+        with sup as one:
+            total += one
+            {}[i]
+        with lock, rlock:
+            with rlock:
+                total += 1
+        with contextlib.suppress(KeyError):
+            {}[i]
+    assert not lock.locked()
+    return total, len(cm.log)
+
+
+assert run_with(3000) == (9000, 6000)
+
+
+# The special-method site follows class changes.
+class Swap:
+    def __enter__(self):
+        return "a"
+
+    def __exit__(self, *a):
+        return False
+
+
+def enter_value(obj):
+    with obj as v:
+        return v
+
+
+for i in range(2000):
+    assert enter_value(Swap()) == "a"
+Swap.__enter__ = lambda self: "b"
+assert enter_value(Swap()) == "b"
+del Swap.__enter__
+try:
+    enter_value(Swap())
+except TypeError as e:
+    assert "context manager" in str(e), e
+else:
+    raise AssertionError("missing __enter__ accepted")
+
+
+# An instance attribute never stands in for the special method.
+class OnInstance:
+    def __enter__(self):
+        return "class"
+
+    def __exit__(self, *a):
+        return False
+
+
+o = OnInstance()
+o.__enter__ = lambda: "instance"
+for _ in range(500):
+    assert enter_value(o) == "class"
+
+
+# An exception in the body reaches __exit__ with its traceback.
+class Seen:
+    def __enter__(self):
+        return self
+
+    def __exit__(self, t, v, tb):
+        self.got = (t, v, tb)
+        return True
+
+
+for _ in range(200):
+    s = Seen()
+    with s:
+        raise ValueError("body")
+    t, v, tb = s.got
+    assert t is ValueError and isinstance(v, ValueError) and tb is v.__traceback__
+
+
+@contextlib.contextmanager
+def tagged(log, tag):
+    log.append(tag)
+    try:
+        yield len(log)
+    finally:
+        log.pop()
+
+
+log = []
+for i in range(500):
+    with tagged(log, i) as k:
+        assert k == 1 and log == [i]
+assert log == []
+print("ok")
diff --git a/tests/regrtest/test_split_attr_delete.py b/tests/regrtest/test_split_attr_delete.py
new file mode 100644
index 00000000..682355d9
--- /dev/null
+++ b/tests/regrtest/test_split_attr_delete.py
@@ -0,0 +1,82 @@
+"""Deleting an instance attribute keeps the split layout (no dictionary
+is materialized), and the instance still behaves exactly like a dict-backed
+one: lookups, `__dict__` order, re-adding, AttributeError, prompt
+finalization, and compiled code's cached field reads."""
+
+
+class O:
+    def __init__(self):
+        self.a = 1
+        self.b = 2
+        self.c = 3
+        self.d = 4
+
+    def total(self):
+        return self.a + self.c + self.d
+
+
+class Fin:
+    log = []
+
+    def __del__(self):
+        Fin.log.append(1)
+
+
+def run():
+    for _ in range(2000):
+        o = O()
+        del o.b
+        assert not hasattr(o, "b")
+        assert getattr(o, "b", None) is None
+        assert o.a == 1 and o.c == 3 and o.d == 4
+        assert o.total() == 8
+        assert list(vars(o)) == ["a", "c", "d"], vars(o)
+        o.b = 5
+        assert list(vars(o).items()) == [("a", 1), ("c", 3), ("d", 4), ("b", 5)]
+        assert o.b == 5
+
+        p = O()
+        del p.a, p.d
+        assert p.__dict__ == {"b": 2, "c": 3}
+        p.c = 7
+        assert p.c == 7 and getattr(p, "a", None) is None
+        try:
+            del p.a
+        except AttributeError as e:
+            assert "'a'" in str(e), e
+        else:
+            raise AssertionError("deleted twice")
+
+        # The last attribute simply leaves.
+        q = O()
+        del q.d
+        q.e = 9
+        assert list(q.__dict__) == ["a", "b", "c", "e"]
+        assert not hasattr(q, "d")
+
+        r = O()
+        del r.a
+        del r.b
+        del r.c
+        del r.d
+        assert r.__dict__ == {}
+        r.z = 1
+        assert r.__dict__ == {"z": 1}
+
+        # A fresh instance of the class keeps its fast layout.
+        s = O()
+        assert s.total() == 8 and list(vars(s)) == ["a", "b", "c", "d"]
+
+    # The deleted value dies at the `del`.
+    o = O()
+    o.b = Fin()
+    del o.b
+    assert Fin.log == [1], Fin.log
+    o.c = Fin()
+    del o.a, o.c
+    assert Fin.log == [1, 1], Fin.log
+    assert vars(o) == {"d": 4}
+
+
+run()
+print("ok")

From 4ef65e1b5078a2bc3e04c2645fd36b3ec5031287 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 19:18:12 -0700
Subject: [PATCH 007/173] perf: run except-clause cleanups and deletes in the
 core loop

The core loop handed `DELETE_ATTR` and `DELETE_FAST` to the full leaf
arms, so contextlib's generator managers (`del self.args, self.kwds,
self.func` on every `__enter__`) and every `except ... as e:` clause's
`del e` paid a slow-arm round trip. Both now have core-loop arms that
run the full arms' common shapes (a plain instance's own attribute, a
bound plain local) and release as they do.

Smaller savings on the same paths:

- Materializing an instance's split attributes into a `__dict__` (as
  `logging`'s `record.__dict__` does per record) moves the values
  instead of copying them first, and inserts each name with the hash its
  shared-names table kept, skipping hashing and comparisons.
- `raise ... from None` stores `__cause__ = None` only over an existing
  cause (an unset one already reads None).
- A traceback chain is torn down without moving each node out of its
  allocation.

Per-iteration instructions: del attr 1188 -> 981, three deletes 6217 ->
5610, contextmanager with 26789 -> 26239. Suite vs. the branch base:
context_managers -7.5%, exceptions -4.1%, raytrace -1.9%, logging_mix
-0.8%, geomean 0.9982.
---
 crates/weavepy-vm/src/inst_dict.rs          |  28 ++++-
 crates/weavepy-vm/src/lib.rs                | 112 +++++++++++++++++++-
 crates/weavepy-vm/src/object.rs             |  11 +-
 tests/regrtest/test_exception_fast_paths.py |  84 +++++++++++++++
 4 files changed, 227 insertions(+), 8 deletions(-)

diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index ca9f8872..ebbcb672 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -610,6 +610,18 @@ impl SplitValues {
             .filter_map(move |(i, v)| Some((keys?.get(i)?, v)))
     }
 
+    /// A new strong handle on the names, while any value is (or was) set.
+    fn keys_rc(&self) -> Option<Rc<SharedKeys>> {
+        self.block.and_then(|b| {
+            // SAFETY: a non-null `keys` is a live strong reference.
+            let k = unsafe { (*b.as_ptr()).keys };
+            (!k.is_null()).then(|| unsafe {
+                Rc::increment_strong_count(k);
+                Rc::from_raw(k)
+            })
+        })
+    }
+
     /// A copy of the names and values (for a materialization that can't
     /// move them).
     fn snapshot(&self) -> (Option<Rc<SharedKeys>>, Vec<Object>) {
@@ -967,7 +979,7 @@ impl InstDict {
         let owner = self.owner();
         let (keys, values) = match self.split.try_borrow_mut() {
             Ok(mut s) => {
-                let keys = s.snapshot().0;
+                let keys = s.keys_rc();
                 (keys, s.take())
             }
             // A live view of the values (nothing in the VM holds one across
@@ -988,12 +1000,22 @@ impl InstDict {
             DictData::with_capacity_and_hasher(n, crate::fasthash::FxBuildHasher)
         };
         if let Some(keys) = keys {
+            use indexmap::map::raw_entry_v1::{RawEntryApiV1, RawEntryMut};
             // A deferred owner holds only atomic values, and a tracked
-            // one needs no barrier: insert without either.
+            // one needs no barrier: insert without either. The names are
+            // distinct `str`s whose hashes the table kept, so each goes
+            // straight into its bucket, without hashing or comparing.
             let map = d.map_mut_atomic_store();
             for (i, v) in values.into_iter().enumerate() {
                 if let Some(k) = keys.get(i) {
-                    map.insert(k.clone(), v);
+                    // SAFETY: slot `i` is published (`get` found its name).
+                    let hash = crate::object::dict_table_hash(unsafe { *keys.hashes[i].get() });
+                    match map.raw_entry_mut_v1().from_hash(hash, |_| false) {
+                        RawEntryMut::Vacant(e) => {
+                            e.insert_hashed_nocheck(hash, k.clone(), v);
+                        }
+                        RawEntryMut::Occupied(_) => unreachable!("never matches"),
+                    }
                 }
             }
         }
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 863f2b54..5a8437d5 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -11639,6 +11639,39 @@ impl Interpreter {
                         last = pc;
                         pc += 1;
                     }
+                    // `del x` of a bound plain local (an `except ... as e:`
+                    // clause's cleanup, after its `e = None`), as the full
+                    // arm runs it: an unbound local raises there, and a slot
+                    // shared with a cell is the full handler's.
+                    OpCode::DeleteFast => {
+                        let i = ins.arg as usize;
+                        if i >= nlocals
+                            || (!code.cellvars.is_empty()
+                                && Self::shared_cell_index(code, i).is_some())
+                        {
+                            break None;
+                        }
+                        // SAFETY: `i < nlocals`.
+                        unsafe {
+                            let slot = lbase.add(i);
+                            if matches!(*slot, Object::Unbound) {
+                                break None;
+                            }
+                            if !scalar(&*slot) {
+                                if !Self::core_droppable(&*slot) {
+                                    break None;
+                                }
+                                self.release(std::mem::replace(&mut *slot, Object::Unbound));
+                                last = pc;
+                                pc += 1;
+                                after_release!();
+                                continue;
+                            }
+                            slot.write(Object::Unbound);
+                        }
+                        last = pc;
+                        pc += 1;
+                    }
                     // A list literal, tracked like the full handler's (an
                     // allocation due to trigger a collection is left to it).
                     OpCode::BuildList => {
@@ -14569,6 +14602,19 @@ impl Interpreter {
                     // leaf arms' shapes, which the full handlers' match).
                     // An `except` clause's entry, test and exit (see
                     // `core_exc_op`).
+                    // `del obj.attr` of a plain instance's own attribute (see
+                    // `core_delete_attr`); anything else is the full arm's.
+                    OpCode::DeleteAttr => {
+                        // SAFETY: the `len` slots at `base` are the core
+                        // loop's initialized stack.
+                        match unsafe { self.core_delete_attr(code, ins, base, len) } {
+                            Some(n) => len = n,
+                            None => break None,
+                        }
+                        last = pc;
+                        pc += 1;
+                        after_release!();
+                    }
                     OpCode::PushExcInfo | OpCode::CheckExcMatch | OpCode::PopExcept => {
                         // SAFETY: the `len` slots at `base` are the core
                         // loop's initialized stack, with room to `cap`.
@@ -17181,6 +17227,66 @@ impl Interpreter {
         }
     }
 
+    /// `del obj.attr` (`DELETE_ATTR` `ins` of `code`) of the instance atop
+    /// the `len` stack slots at `base`, as the full leaf arm runs it: an
+    /// ordinary instance's own attribute, by the default `__delattr__`,
+    /// with no class attribute of that name. The value and the instance
+    /// leave as the full arm releases them (which may queue a finalizer:
+    /// the caller checks). Returns the new stack length, or `None`, having
+    /// touched nothing (a missing attribute raises through the full arm).
+    ///
+    /// # Safety
+    ///
+    /// The `len` slots at `base` are initialized operand stack entries.
+    #[inline(never)]
+    unsafe fn core_delete_attr(
+        &mut self,
+        code: &CodeObject,
+        ins: weavepy_compiler::Instruction,
+        base: *mut Object,
+        len: usize,
+    ) -> Option<usize> {
+        if len == 0 {
+            return None;
+        }
+        // SAFETY: `len > 0`.
+        let Object::Instance(inst) = (unsafe { &*base.add(len - 1) }) else {
+            return None;
+        };
+        let Some(Object::Str(key)) = code_name_obj(code, ins.arg) else {
+            return None;
+        };
+        let cls = inst.cls_raw();
+        // (The full arm's conditions; see there.)
+        if inst.native.get().is_some()
+            || key.starts_with("__")
+            || &**key == "_CHUNK_SIZE"
+            || cls.flags.is_exception
+            || cls.forbids_dict
+            || crate::capi_watchers::dicts_active()
+            || !Self::default_delattr(cls)
+            || !matches!(
+                Self::leaf_class_attr(code, cls, ins.arg),
+                Some(LeafAttr::InstanceOnly)
+            )
+        {
+            return None;
+        }
+        let removed = match inst.split_remove(key) {
+            Some(removed) => removed?,
+            None => inst
+                .dict_cell()
+                .borrow_mut()
+                .shift_remove(&DictKey(Object::Str(key.clone())))?,
+        };
+        // SAFETY: the instance leaves its slot.
+        let owner = unsafe { base.add(len - 1).read() };
+        // Released at once, as CPython's decref is.
+        self.release(removed);
+        self.release(owner);
+        Some(len - 1)
+    }
+
     /// Zero-argument `super().name` (`LOAD_SUPER_ATTR` `ins` at `pc` over
     /// the `len` stack slots at `base`; top-down: self, class, the `super`
     /// global) of a plain method later in the receiver's MRO, in the
@@ -31598,7 +31704,11 @@ impl Interpreter {
             if matches!(c, Object::None) {
                 pe.cause = None;
                 if let Object::Instance(ref inst_rc) = pe.instance {
-                    inst_rc.slot_set("__cause__", Object::None);
+                    // An unset `__cause__` already reads None (a re-raised
+                    // instance's earlier cause is cleared).
+                    if inst_rc.slots.borrow().get("__cause__").is_some() {
+                        inst_rc.slot_set("__cause__", Object::None);
+                    }
                     inst_rc.slot_set("__suppress_context__", Object::Bool(true));
                 }
             } else {
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index feff05d4..c182d895 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -1522,12 +1522,15 @@ impl Drop for PyTraceback {
         // is a `tb_next` linked list one node per stack level, so a
         // ~1000-deep traceback (a `RecursionError`) would drop recursively
         // and overflow the native stack. Tear the chain down iteratively.
+        // (The last owner of each link detaches its successor in place,
+        // then frees a node with nothing left to chain to.)
         let mut link = self.next.borrow_mut().take();
         while let Some(tb) = link {
-            match Rc::try_unwrap(tb) {
-                Ok(inner) => link = inner.next.borrow_mut().take(),
-                Err(_) => break,
+            if Rc::strong_count(&tb) != 1 {
+                break;
             }
+            link = tb.next.borrow_mut().take();
+            drop(tb);
         }
     }
 }
@@ -4021,7 +4024,7 @@ pub(crate) fn dict_key_is_reentrant(key: &Object) -> bool {
 
 /// The table-level (Fx-mixed) hash for a Python hash value, matching what
 /// `DictKey::hash` feeds the [`DictData`] hasher.
-fn dict_table_hash(py_hash: i64) -> u64 {
+pub(crate) fn dict_table_hash(py_hash: i64) -> u64 {
     use std::hash::BuildHasher;
     crate::fasthash::FxBuildHasher.hash_one(py_hash)
 }
diff --git a/tests/regrtest/test_exception_fast_paths.py b/tests/regrtest/test_exception_fast_paths.py
index 898ac4c0..d1b14f7b 100644
--- a/tests/regrtest/test_exception_fast_paths.py
+++ b/tests/regrtest/test_exception_fast_paths.py
@@ -254,6 +254,90 @@ def __exit__(self, t, v, tb):
     assert t is ValueError and isinstance(v, ValueError) and tb is v.__traceback__
 
 
+# `raise ... from None` clears an earlier cause and suppresses the context.
+class AppError(Exception):
+    pass
+
+
+def chained(e):
+    try:
+        {}["k"]
+    except KeyError:
+        raise e from None
+
+
+for _ in range(300):
+    err = AppError("x")
+    assert err.__cause__ is None and err.__suppress_context__ is False
+    try:
+        chained(err)
+    except AppError as caught:
+        assert caught.__cause__ is None
+        assert caught.__suppress_context__ is True
+        assert isinstance(caught.__context__, KeyError)
+    err.__cause__ = ValueError("old")
+    try:
+        chained(err)
+    except AppError as caught:
+        assert caught.__cause__ is None, caught.__cause__
+
+
+# The `as` name's cleanup (`e = None; del e`) unbinds it.
+def bound_after():
+    try:
+        raise ValueError
+    except ValueError as e:
+        pass
+    try:
+        e
+    except UnboundLocalError:
+        return True
+    return False
+
+
+for _ in range(300):
+    assert bound_after()
+
+
+# A traceback chain across frames survives its exception's handler while
+# referenced, and goes when the last reference does.
+def deep(n):
+    if n == 0:
+        raise KeyError("deep")
+    deep(n - 1)
+
+
+kept = []
+for i in range(200):
+    try:
+        deep(3)
+    except KeyError as e:
+        if i % 50 == 0:
+            kept.append(e.__traceback__)
+for tb in kept:
+    depth = 0
+    while tb is not None:
+        depth += 1
+        tb = tb.tb_next
+    assert depth == 5, depth
+
+
+# An instance's `__dict__` after its split layout keeps names and order.
+class Many:
+    def __init__(self):
+        for i in range(20):
+            setattr(self, "a%d" % i, i)
+
+
+for _ in range(200):
+    m = Many()
+    d = m.__dict__
+    assert list(d) == ["a%d" % i for i in range(20)]
+    assert d["a7"] == 7 and m.a19 == 19
+    d["a7"] = 70
+    assert m.a7 == 70
+
+
 @contextlib.contextmanager
 def tagged(log, tag):
     log.append(tag)

From 616e3eb8d350b53f1c066aef26c0e41447c07a79 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 20:00:24 -0700
Subject: [PATCH 008/173] perf: load instance methods in line in compiled code

A method load on an instance now runs in the frame JIT's code when the
site's cache holds the instance's class version: the cached plain
function, its count bumped in place, replaces the receiver, which moves
above it. An instance can't shadow the method when it has no published
__dict__ and holds fewer split values than the name's position among
its class's shared names (the names only ever grow), so the check is a
few loads and compares. Anything else takes h_load_method, which fills
the cache when it resolves a plain function on an instance that doesn't
shadow it.

Python functions now count among the variants whose clone and release
compiled code makes a plain count update.

Instructions retired: go -5.0%; p_meth probe -6%.
---
 crates/weavepy-vm/src/frame_jit.rs | 186 ++++++++++++++++++++++++++---
 1 file changed, 172 insertions(+), 14 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 6b208522..14a5a411 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -193,6 +193,9 @@ pub(crate) struct Native {
     _globals: Vec<Box<GlobalCache>>,
     #[allow(clippy::vec_box)]
     _slots: Vec<Box<SlotCache>>,
+    /// The method-load sites' in-line caches.
+    #[allow(clippy::vec_box)]
+    _methods: Vec<Box<MethodCache>>,
     /// The `CALL` sites' direct-call caches.
     #[allow(clippy::vec_box)]
     _direct: Vec<Box<DirectSite>>,
@@ -781,6 +784,18 @@ fn tags() -> Option<Tags> {
         unsafe { *std::ptr::from_ref(v).cast::<u8>().add(1) }
     };
     let cell = Object::Cell(Rc::new(crate::sync::RefCell::new(Object::None)));
+    let func = {
+        let dict = Rc::new(crate::sync::RefCell::new(crate::object::DictData::default()));
+        crate::new_function(
+            Rc::new(CodeObject::default()),
+            &dict,
+            &dict,
+            Vec::new(),
+            Vec::new(),
+            Vec::new(),
+            None,
+        )
+    };
     let inst = Object::Instance(Rc::new(crate::types::PyInstance::new(
         crate::builtin_types::builtin_types().object_.clone(),
     )));
@@ -802,18 +817,7 @@ fn tags() -> Option<Tags> {
         float: tag(&Object::Float(1.0)),
         cell: tag(&cell),
         instance: tag(&inst),
-        function: {
-            let dict = Rc::new(crate::sync::RefCell::new(crate::object::DictData::default()));
-            tag(&crate::new_function(
-                Rc::new(CodeObject::default()),
-                &dict,
-                &dict,
-                Vec::new(),
-                Vec::new(),
-                Vec::new(),
-                None,
-            ))
-        },
+        function: tag(&func),
         list: tag(&Object::new_list(Vec::new())),
         list_layout: list_layout(),
         tuple: tag(&Object::new_tuple_array([Object::None])),
@@ -827,6 +831,7 @@ fn tags() -> Option<Tags> {
                 &Object::new_list(Vec::new()),
                 &Object::new_dict(),
                 &Object::Type(crate::builtin_types::builtin_types().object_.clone()),
+                &func,
             ]
             .into_iter()
             .filter(|o| tag(o) < 64 && counted_at_payload(o, word(o)))
@@ -1957,6 +1962,7 @@ unsafe extern "C" fn h_load_method(
     ext: *const CodeConstObjects,
     pc: u64,
     len: u64,
+    cache: *mut MethodCache,
 ) -> u32 {
     // SAFETY: the code passes its live state, its own code and extension,
     // one of its method loads, and its stack depth (at least one, with
@@ -2001,7 +2007,12 @@ unsafe extern "C" fn h_load_method(
                     return u32::from(!Interpreter::core_value_method(code, name, top));
                 }
                 let f = match ms.get_held(ver) {
-                    Some(f) => Object::Function(f),
+                    Some(f) => {
+                        if !cache.is_null() {
+                            fill_method_cache(&mut *cache, cls, ver, &f, ext, name);
+                        }
+                        Object::Function(f)
+                    }
                     None => match ms.get_inst_builtin(ver) {
                         Some(b) => Object::Builtin(b),
                         None => match ext.name_objs.get(name as usize).and_then(|n| match n {
@@ -2052,6 +2063,64 @@ unsafe extern "C" fn h_load_method(
     }
 }
 
+/// A method-load site's in-line cache (see `Lower::load_method`): the
+/// class version (`0` empty) whose plain function `func` (its payload
+/// word) the site loaded, and what proves an instance doesn't shadow it:
+/// the class's shared names `keys` (`0` none) and the name's position
+/// among them (an instance holding fewer values can't hold the name).
+#[repr(C)]
+#[derive(Default)]
+pub(crate) struct MethodCache {
+    ver: u64,
+    func: u64,
+    keys: u64,
+    before: u32,
+}
+
+const MC_VER: i32 = std::mem::offset_of!(MethodCache, ver) as i32;
+const MC_FUNC: i32 = std::mem::offset_of!(MethodCache, func) as i32;
+const MC_KEYS: i32 = std::mem::offset_of!(MethodCache, keys) as i32;
+const MC_BEFORE: i32 = std::mem::offset_of!(MethodCache, before) as i32;
+
+/// Record in `cache` that instances of `cls` at version `ver` load the
+/// plain function `f` for the name `co_names[name]` (the caller proved
+/// this instance doesn't shadow it, and that `cls` keeps the default
+/// `__getattribute__`).
+fn fill_method_cache(
+    cache: &mut MethodCache,
+    cls: &crate::types::TypeObject,
+    ver: u64,
+    f: &Rc<crate::object::PyFunction>,
+    ext: &CodeConstObjects,
+    name: u32,
+) {
+    let (keys, before) = match cls.shared_keys.get() {
+        Some(keys) => {
+            let (Some(Object::Str(n)), Some(&hash)) = (
+                ext.name_objs.get(name as usize),
+                ext.name_hashes.get(name as usize),
+            ) else {
+                return;
+            };
+            (
+                std::ptr::from_ref(keys) as u64,
+                u32::try_from(keys.names_before(n, hash)).unwrap_or(u32::MAX),
+            )
+        }
+        None => (0, 0),
+    };
+    let obj = Object::Function(f.clone());
+    // SAFETY: an `Object` is 16 bytes; its second word is the payload.
+    let word = unsafe { std::ptr::from_ref(&obj).cast::<u64>().add(1).read() };
+    drop(obj);
+    *cache = MethodCache {
+        ver,
+        func: word,
+        keys,
+        before,
+    };
+}
+
 /// A `LOAD_ATTR` site's `__slots__` member shortcut: the class version and
 /// the member's position the site's cache last proved (`ver` 0 empty),
 /// read without decoding the site's cache.
@@ -3826,6 +3895,7 @@ fn compile_with(
     let mut entries: Vec<bool>;
     let mut globals: Vec<Box<GlobalCache>>;
     let mut slots: Vec<Box<SlotCache>>;
+    let mut methods: Vec<Box<MethodCache>>;
     let mut direct: Vec<Box<DirectSite>>;
     let t0 = stats::enabled().then(std::time::Instant::now);
     // A body whose code comes out large is lowered again lean: compiling
@@ -3840,6 +3910,7 @@ fn compile_with(
         entries = (0..ninstrs).map(|pc| lower.enters_at(pc)).collect();
         globals = std::mem::take(&mut lower.global_caches);
         slots = std::mem::take(&mut lower.slot_caches);
+        methods = std::mem::take(&mut lower.method_caches);
         direct = std::mem::take(&mut lower.direct_sites);
         if done {
             lower.b.seal_all_blocks();
@@ -3900,6 +3971,7 @@ fn compile_with(
         need: stack_need(depths),
         _globals: globals,
         _slots: slots,
+        _methods: methods,
         _direct: direct,
     })
 }
@@ -4001,6 +4073,7 @@ struct Lower<'a> {
     global_caches: Vec<Box<GlobalCache>>,
     #[allow(clippy::vec_box)]
     slot_caches: Vec<Box<SlotCache>>,
+    method_caches: Vec<Box<MethodCache>>,
     #[allow(clippy::vec_box)]
     direct_sites: Vec<Box<DirectSite>>,
     /// The block a helper's raise goes to (see [`Self::leave`]), made on
@@ -4161,6 +4234,7 @@ impl<'a> Lower<'a> {
             side_exits: Vec::new(),
             global_caches: Vec::new(),
             slot_caches: Vec::new(),
+            method_caches: Vec::new(),
             direct_sites: Vec::new(),
             catch: None,
         }
@@ -7009,19 +7083,103 @@ impl<'a> Lower<'a> {
             return false;
         }
         self.flush();
+        let helper = self.b.create_block();
+        let done = self.b.create_block();
+        let fn_counted = self.tags.rc_counted & (1i64 << self.tags.function) != 0;
+        let cache: Box<MethodCache> = Box::default();
+        let cache_at = std::ptr::from_ref::<MethodCache>(&cache) as i64;
+        self.method_caches.push(cache);
+        match self.layout {
+            // An instance of the class the site last saw, which holds no
+            // dictionary and fewer values than the name's position among
+            // its class's names: the cached function, the receiver above
+            // it.
+            Some(l) if fn_counted => {
+                let l = *l;
+                let ptr = self.ptr;
+                let recv = self.slot_addr(self.depth - 1);
+                let above = self.slot_addr(self.depth);
+                let c = self.b.ins().iconst(ptr, cache_at);
+                let tag = self.tag_at(recv);
+                let not_inst =
+                    self.b
+                        .ins()
+                        .icmp_imm(IntCC::NotEqual, tag, i64::from(self.tags.instance));
+                self.branch_out(not_inst, helper);
+                let inst = self.b.ins().load(ptr, FLAGS, recv, 8);
+                let cls = self.b.ins().load(ptr, FLAGS, inst, l.inst_class);
+                let ver = self
+                    .b
+                    .ins()
+                    .load(types::I64, FLAGS, cls, l.type_attr_version);
+                let want = self.b.ins().load(types::I64, FLAGS, c, MC_VER);
+                let lazy = self.b.ins().load(ptr, FLAGS, inst, l.inst_dict_lazy);
+                let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64);
+                let unguarded = self.b.ins().uload8(types::I64, FLAGS, flag, 0);
+                let rflag = self.b.ins().iconst(
+                    ptr,
+                    std::ptr::from_ref(crate::sync::rc_shared_flag()) as i64,
+                );
+                let rshared = self.b.ins().uload8(types::I64, FLAGS, rflag, 0);
+                let borrow = self.b.ins().sload32(FLAGS, inst, l.inst_split_borrow);
+                let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0);
+                let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want);
+                let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0);
+                let other = self.b.ins().bor(lazy, unguarded);
+                let other = self.b.ins().bor(other, rshared);
+                let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0);
+                let bad = self.b.ins().bor(unset, stale);
+                let bad = self.b.ins().bor(bad, busy);
+                let bad = self.b.ins().bor(bad, other);
+                self.branch_out(bad, helper);
+                let block = self.b.ins().load(ptr, FLAGS, inst, l.inst_split_block);
+                let check = self.b.create_block();
+                let hit = self.b.create_block();
+                self.b.ins().brif(block, check, &[], hit, &[]);
+                self.b.switch_to_block(check);
+                let n = self.b.ins().uload32(FLAGS, block, l.split_len);
+                let keys = self.b.ins().load(types::I64, FLAGS, block, l.split_keys);
+                let ckeys = self.b.ins().load(types::I64, FLAGS, c, MC_KEYS);
+                let before = self.b.ins().uload32(FLAGS, c, MC_BEFORE);
+                let empty = self.b.ins().icmp_imm(IntCC::Equal, n, 0);
+                let same = self.b.ins().icmp(IntCC::Equal, keys, ckeys);
+                let short = self.b.ins().icmp(IntCC::UnsignedLessThanOrEqual, n, before);
+                let held = self.b.ins().band(same, short);
+                let ok = self.b.ins().bor(empty, held);
+                self.b.ins().brif(ok, hit, &[], helper, &[]);
+                self.b.switch_to_block(hit);
+                // The function's count, then the receiver up and the
+                // function in its place.
+                let f = self.b.ins().load(types::I64, FLAGS, c, MC_FUNC);
+                let count = self.b.ins().load(types::I64, FLAGS, f, 0);
+                let count = self.b.ins().iadd_imm(count, 1);
+                self.b.ins().store(FLAGS, count, f, 0);
+                self.copy16(above, recv);
+                self.write_tag(recv, self.tags.function);
+                self.b.ins().store(FLAGS, f, recv, 8);
+                self.b.ins().jump(done, &[]);
+            }
+            _ => {
+                self.b.ins().jump(helper, &[]);
+            }
+        }
+        self.b.switch_to_block(helper);
         let code = self.code_ptr();
         let ext = self.ext_ptr();
         let pcv = self.b.ins().iconst(types::I64, pc as i64);
         let len = self.b.ins().iconst(types::I64, self.depth as i64);
+        let c = self.b.ins().iconst(self.ptr, cache_at);
         let r = self
             .call(
                 h_load_method as *const () as usize,
-                &[self.st, code, ext, pcv, len],
+                &[self.st, code, ext, pcv, len, c],
                 true,
             )
             .expect("returns");
         let out = self.exit_with(pc, &[], INTERP);
         self.branch_out(r, out);
+        self.b.ins().jump(done, &[]);
+        self.b.switch_to_block(done);
         self.depth += 1;
         self.set_last(pc);
         self.check_released(pc + 1);

From 648a08b183c5118fdb41f090b9009f5067ae3582 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 20:41:15 -0700
Subject: [PATCH 009/173] perf: remember where a method name sits among a
 class's names

Checking that an instance's split values don't shadow a method scanned
the class's shared names on every pure-leaf method call whenever the
names' hash filter couldn't rule the name out, which depends on the
process's hash seed: with an unlucky seed the scan was 10% of go. Each
code name now remembers the names table it last searched and the
name's position there; the table only grows, so values no longer than
that position can't hold the name and need no scan.
---
 crates/weavepy-vm/src/inst_dict.rs | 39 ++++++++++++++++++++++++++++++
 crates/weavepy-vm/src/lib.rs       | 17 +++++++++++--
 2 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index ebbcb672..9bf1f2e4 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -214,6 +214,18 @@ fn key_names_str(key: &DictKey, name: &str) -> bool {
     matches!(&key.0, Object::Str(s) if name_eq(s, name))
 }
 
+/// A name's last position among a class's shared names (see
+/// [`SplitValues::holds_memo`]): the names table's address and the
+/// position. Touched only under the GIL.
+#[derive(Default, Debug)]
+pub struct NameMemo(std::cell::UnsafeCell<(usize, u32)>);
+
+// SAFETY: read and written only under the GIL (never in free-threaded
+// mode, whose paths don't consult it).
+unsafe impl Sync for NameMemo {}
+// SAFETY: as above.
+unsafe impl Send for NameMemo {}
+
 /// The header of an instance's split values allocation; the values
 /// follow it.
 #[repr(C)]
@@ -573,6 +585,33 @@ impl SplitValues {
         self.keys()?.position_hashed(self.len(), name, hash)
     }
 
+    /// Whether these values hold `name` (Python hash `hash`), with `memo`
+    /// remembering the names table it last searched and the name's
+    /// position there (all of them when absent). The table only grows,
+    /// so values no longer than that position can't hold the name, and
+    /// only longer ones search again.
+    #[inline]
+    pub fn holds_memo(&self, name: &str, hash: i64, memo: &NameMemo) -> bool {
+        let Some(keys) = self.keys() else {
+            return false;
+        };
+        // (A clear filter bit settles it at once.)
+        if keys.filter.load(Ordering::Relaxed) & (1 << (hash & 63)) == 0 {
+            return false;
+        }
+        let n = self.len();
+        let at = std::ptr::from_ref(keys) as usize;
+        // SAFETY: GIL-serialized (see `NameMemo`).
+        let (k, before) = unsafe { *memo.0.get() };
+        if k == at && n <= before as usize {
+            return false;
+        }
+        let before = keys.names_before(name, hash);
+        // SAFETY: as above.
+        unsafe { *memo.0.get() = (at, u32::try_from(before).unwrap_or(u32::MAX)) };
+        n > before
+    }
+
     /// [`Self::position`] for a borrowed name (identity of the bytes
     /// first: a name read off an interned string settles without a
     /// comparison).
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 5a8437d5..37d357eb 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -70628,6 +70628,9 @@ struct CodeConstObjects {
     /// The body's store-only `__init__` shape (see [`code_store_init`]),
     /// or `None` when it isn't one.
     store_init: std::sync::OnceLock<Option<Box<StoreInit>>>,
+    /// Per `co_names` entry, its position among the shared names of the
+    /// instances last checked for shadowing it (see [`inst_may_shadow`]).
+    name_memos: std::sync::OnceLock<Box<[crate::inst_dict::NameMemo]>>,
     /// The code's name and qualified name as interned strings: a fresh
     /// generator's `gi_name` and `gi_qualname` while its function's names
     /// are untouched.
@@ -72722,6 +72725,7 @@ fn code_vm_ext_build(code: &CodeObject) -> &CodeConstObjects {
             leaf_plan: std::sync::OnceLock::new(),
             returns_none: std::sync::atomic::AtomicU8::new(0),
             store_init: std::sync::OnceLock::new(),
+            name_memos: std::sync::OnceLock::new(),
             gen_names: std::sync::OnceLock::new(),
             #[cfg(feature = "jit")]
             frame_jit: frame_jit::Slot::default(),
@@ -73071,9 +73075,18 @@ fn inst_may_shadow(inst: &PyInstance, code: &CodeObject, name_idx: u32) -> bool
                 return false;
             }
             let i = name_idx as usize;
-            match code_vm_ext(code).map(|t| (t.name_objs.get(i), t.name_hashes.get(i))) {
+            let ext = code_vm_ext(code);
+            match ext.map(|t| (t.name_objs.get(i), t.name_hashes.get(i))) {
                 Some((Some(Object::Str(n)), Some(&hash))) => {
-                    split.position_hashed(n, hash).is_some()
+                    let memos = ext.expect("matched above").name_memos.get_or_init(|| {
+                        (0..code.names.len())
+                            .map(|_| crate::inst_dict::NameMemo::default())
+                            .collect()
+                    });
+                    match memos.get(i) {
+                        Some(memo) => split.holds_memo(n, hash, memo),
+                        None => split.position_hashed(n, hash).is_some(),
+                    }
                 }
                 _ => code_name_key(code, name_idx)
                     .is_none_or(|k| split.position_hashed(k.s, k.hash).is_some()),

From e0465de1bd31488a5f1a781b88415976974061bc Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 18:30:18 -0700
Subject: [PATCH 010/173] perf: compile coroutine bodies and step closure
 genexprs natively

Coroutine and `yield from` bodies now compile in the frame JIT:
`GET_AWAITABLE`, `GET_YIELD_FROM_ITER`, `SEND` and `END_SEND` run
through helpers that do what the core loop's arms do (`SEND` switches
to the delegate and reloads), and a generator-family body warms at its
creation, since its activations never start at pc 0 where a call's heat
is counted. `fib` in the coroutines benchmark and `Tree.__iter__` in
generators_tree now run natively.

Generator expressions over closures and subscripts take fast steps once
their bodies compile (`gen_fast`'s CODE_NATIVE verdict): `LOAD_DEREF`,
`BINARY_SUBSCR`, `LOAD_ATTR`, `IS_OP` and a few others are left to the
native code, whose helpers decline instead of raising in a fast step;
a release that queues a finalizer now ends the step so the general loop
runs it first. Exhausted tuple iterators, and list iterators a
generator expression's `.0` shares, retire in the native and fast-step
`FOR_ITER` as the core loop's arm retires them, instead of handing the
generator's end to the general loop.

Generator creation is leaner: the `RETURN_GENERATOR` scan is cached in
the code's extension, the frame of a cell-free body is built in place
with the arguments moved straight into its locals, the frame's weak
backlink is stamped through `Rc::new_cyclic`, the coroutine origin
depth skips its thread-local until some thread sets it, and the core
loop goes on after making a generator instead of reloading. The frame
JIT's loop heuristic no longer counts a generator expression's creating
call, or the cold fallback of an inlined `tuple`/`list`/`all`/`any`, as
exits, so consumer loops like `t += len(tuple(x for x in v))` compile.

Instructions retired vs the branch point (suite, per call): nqueens
0.887, coroutines 0.925, generators_tree 0.952, async_tree 0.992,
geomean 0.995. Probes: tuple(genexpr) 3.00x -> 2.28x CPython, closure
genexprs (nqueens shape) 2.87x -> 2.31x, recursive await 3.19x ->
2.72x.
---
 crates/weavepy-vm/src/frame_jit.rs            | 232 +++++++++++++++--
 crates/weavepy-vm/src/gen_fast.rs             | 158 +++++++++---
 crates/weavepy-vm/src/lib.rs                  | 237 ++++++++++++++----
 crates/weavepy-vm/src/stdlib/sys.rs           |  12 +
 tests/regrtest/test_await_native_bodies.py    | 194 ++++++++++++++
 .../test_generator_closure_fast_steps.py      | 156 ++++++++++++
 6 files changed, 901 insertions(+), 88 deletions(-)
 create mode 100644 tests/regrtest/test_await_native_bodies.py
 create mode 100644 tests/regrtest/test_generator_closure_fast_steps.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 14a5a411..3bb84e68 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -463,6 +463,37 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
             }
         }
     }
+    // A generator expression's creation (`MAKE_FUNCTION`, the iterable,
+    // `GET_ITER`, `CALL 0`) runs in place through the call helper.
+    for pc in 1..ins.len() {
+        if ins[pc].op == OpCode::Call && ins[pc].arg == 0 && ins[pc - 1].op == OpCode::GetIter {
+            let made = (pc.saturating_sub(16)..pc)
+                .rev()
+                .find(|&k| ins[k].op == OpCode::MakeFunction)
+                .filter(|&k| k > 0 && ins[k - 1].op == OpCode::LoadConst)
+                .and_then(|k| code.constants.get(ins[k - 1].arg as usize));
+            if matches!(made, Some(weavepy_compiler::Constant::Code(c)) if c.is_generator) {
+                helped[pc] = true;
+            }
+        }
+    }
+    // The fallback of an inlined `tuple(...)`, `list(...)`, `all(...)` or
+    // `any(...)` of a generator expression (`LOAD_COMMON_CONSTANT; IS_OP;
+    // POP_JUMP_IF_FALSE` to the plain call, which the inlined loop jumps
+    // over) runs only when the name is rebound: its calls don't count.
+    let mut cold = vec![false; ins.len()];
+    for pc in 0..ins.len().saturating_sub(2) {
+        if ins[pc].op == OpCode::LoadCommonConstant
+            && ins[pc + 1].op == OpCode::IsOp
+            && ins[pc + 2].op == OpCode::PopJumpIfFalse
+        {
+            let from = pc + 3 + ins[pc + 2].arg as usize;
+            if from > pc + 3 && from <= ins.len() && ins[from - 1].op == OpCode::JumpForward {
+                let to = (from + ins[from - 1].arg as usize).min(ins.len());
+                cold[from..to].iter_mut().for_each(|c| *c = true);
+            }
+        }
+    }
     let t = tuning();
     // What running an instruction natively saves, in halves of an
     // in-line instruction's: one run in line saves its whole dispatch and
@@ -513,7 +544,11 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
             | OpCode::PopExcept
             | OpCode::DeleteFast
             | OpCode::StoreSlice
-            | OpCode::DeleteSubscr => false,
+            | OpCode::DeleteSubscr
+            | OpCode::GetAwaitable
+            | OpCode::GetYieldFromIter
+            | OpCode::Send
+            | OpCode::EndSend => false,
             _ => true,
         };
         Some(if in_line { 2 } else { 1 })
@@ -556,6 +591,9 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
         let top = (pc + 1).saturating_sub(i.arg as usize);
         let (mut n, mut exits) = (0usize, 0usize);
         for (k, ins_k) in ins.iter().enumerate().take(pc + 1).skip(top) {
+            if cold[k] {
+                continue;
+            }
             match gain(k) {
                 Some(g) => n += g,
                 // (A loop's epilogue inside a region a handler's back edge
@@ -568,7 +606,10 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
                 // A resume ends at its yield whether or not the body runs
                 // natively: no round trip to pay for.
                 None if ins_k.op == OpCode::YieldValue
-                    && (matches!(at, Heat::Step) || code.is_generator) => {}
+                    && (matches!(at, Heat::Step)
+                        || code.is_generator
+                        || code.is_coroutine
+                        || code.is_async_generator) => {}
                 None => exits += 1,
             }
         }
@@ -1166,7 +1207,14 @@ unsafe extern "C" fn h_for_iter(st: *mut State, it: *mut Object, out: *mut Objec
                     *index += 1;
                     v
                 }
-                None if unique && owner.is_none() => {
+                // A shared iterator (a generator expression's `.0`, which
+                // its local holds too) detaches from the list, as the core
+                // loop's arm does, so a later append can't resurrect it
+                // (when the list lives on: its release frees nothing).
+                None if owner.is_none() && (unique || crate::sync::Rc::strong_count(items) > 1) => {
+                    if !unique {
+                        *items = crate::sync::Rc::new(crate::sync::RefCell::new(Vec::new()));
+                    }
                     // SAFETY: as above.
                     drop(unsafe { it.read() });
                     return 2;
@@ -1174,21 +1222,22 @@ unsafe extern "C" fn h_for_iter(st: *mut State, it: *mut Object, out: *mut Objec
                 None => return 3,
             }
         }
-        PyIterator::Tuple { items, index } => match items.get(*index) {
-            Some(v) => {
-                let v = v.clone();
-                *index += 1;
-                v
-            }
-            // The exhausted iterator leaves the stack, as for a list: its
-            // tuple has another owner, so nothing dies with it.
-            None if unique && crate::object::SharedTuple::strong_count(items) > 1 => {
-                // SAFETY: as above.
+        PyIterator::Tuple { items, index } => {
+            let Some(v) = items.get(*index).cloned() else {
+                // The exhausted iterator leaves the stack (as the core
+                // loop's arm retires it), when its release can't free the
+                // tuple (whose items could queue finalizers): a shared
+                // iterator stays exhausted, the tuple immutable.
+                if unique && crate::shared_value::ThinArc::strong_count(items) == 1 {
+                    return 3;
+                }
+                // SAFETY: the iterator leaves the stack.
                 drop(unsafe { it.read() });
                 return 2;
-            }
-            None => return 3,
-        },
+            };
+            *index += 1;
+            v
+        }
         // `reversed(xs)`: the item below the cursor.
         PyIterator::Reversed { items, index, .. } => {
             if *index < 0 {
@@ -3462,6 +3511,99 @@ unsafe extern "C" fn h_return(st: *mut State, pc: u64, len: u64) -> u32 {
     }
 }
 
+/// `GET_AWAITABLE` (`kind` 0) or `GET_YIELD_FROM_ITER` (`kind` 1) of the
+/// value at `at` (the stack's top), as the core loop's arms run them: `0`
+/// the delegate in place (a coroutine nothing has started, a generator,
+/// or an instance's generator `__await__` / `__iter__`, made here),
+/// anything else declined untouched.
+unsafe extern "C" fn h_await_iter(st: *mut State, at: *mut Object, kind: u64) -> u32 {
+    // SAFETY: the code passes its live state and its initialized top slot.
+    unsafe {
+        let st = &*st;
+        let top = &*at;
+        let ok = if kind == 0 {
+            match top {
+                // A read between instructions.
+                Object::Coroutine(g) => g
+                    .state
+                    .peek()
+                    .is_some_and(|s| matches!(s, crate::object::GeneratorState::Created(_))),
+                // A `types.coroutine` generator is its own.
+                Object::Generator(g) => {
+                    matches!(&g.code, Object::Code(c) if c.is_iterable_coroutine)
+                }
+                _ => false,
+            }
+        } else {
+            let code = &(*st.frame).code;
+            match top {
+                Object::Generator(_) => true,
+                Object::Coroutine(_) => code.is_coroutine || code.is_iterable_coroutine,
+                _ => false,
+            }
+        };
+        if ok {
+            return 0;
+        }
+        if matches!(top, Object::Instance(_)) && Interpreter::core_droppable(top) {
+            let name = if kind == 0 { "__await__" } else { "__iter__" };
+            if let Some(gen) = (*st.interp).instance_gen_call(top, name) {
+                // The instance (droppable) is replaced in place.
+                crate::drop_hot(std::mem::replace(&mut *at, gen));
+                return 0;
+            }
+        }
+        1
+    }
+}
+
+/// `SEND` at `pc` of the running activation, its stack `len` deep: the
+/// generator or coroutine below the top resumed and switched to as the
+/// core loop's arm does (`RELOAD`); anything else (another delegate, or
+/// a generator's fast step, which never switches) declined untouched.
+unsafe extern "C" fn h_send(st: *mut State, pc: u64, len: u64) -> u32 {
+    // SAFETY: the code passes its live state and its stack depth (the
+    // delegate and the sent value on top).
+    unsafe {
+        let st = &mut *st;
+        let len = len as usize;
+        if st.sw.is_null()
+            || len < 2
+            || !matches!(
+                &*st.stack.add(len - 2),
+                Object::Generator(_) | Object::Coroutine(_)
+            )
+        {
+            return 1;
+        }
+        let sw = &mut *st.sw;
+        let frame = &mut *st.frame;
+        frame.stack.set_len(len);
+        frame.pc = pc as u32;
+        *sw.last = st.last;
+        let interp = &mut *st.interp.cast_mut();
+        if !interp.core_gen_resume(sw, pc as usize, crate::InlineResume::Send) {
+            sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+        }
+        RELOAD
+    }
+}
+
+/// `END_SEND` on the slots at `at` (the delegate) and above it (the
+/// result): the result moves down over the delegate, which leaves by a
+/// plain decrement (`0`); anything else declined untouched.
+unsafe extern "C" fn h_end_send(at: *mut Object) -> u32 {
+    // SAFETY: the code passes two initialized slots atop its stack.
+    unsafe {
+        if !Interpreter::core_droppable(&*at) {
+            return 1;
+        }
+        let v = at.add(1).read();
+        crate::drop_hot(std::mem::replace(&mut *at, v));
+        0
+    }
+}
+
 /// `LOAD_DEREF i` of the running frame, as the core loop's arm runs it:
 /// `0` the cell's value written to the free slot `dst`, anything else
 /// declined untouched.
@@ -4156,6 +4298,10 @@ fn native_op(op: OpCode) -> bool {
             | OpCode::DeleteFast
             | OpCode::StoreSlice
             | OpCode::DeleteSubscr
+            | OpCode::GetAwaitable
+            | OpCode::GetYieldFromIter
+            | OpCode::Send
+            | OpCode::EndSend
     )
 }
 
@@ -4766,6 +4912,11 @@ impl<'a> Lower<'a> {
                 OpCode::JumpBackward => Some(next.saturating_sub(ins.arg as usize)),
                 _ => None,
             };
+            // A `SEND`'s exit (`END_SEND`), where a delegate's return
+            // resumes the activation.
+            if ins.op == OpCode::Send && next + (ins.arg as usize) < n {
+                starts[next + ins.arg as usize] = true;
+            }
             if let Some(t) = target {
                 if t < n {
                     starts[t] = true;
@@ -5328,6 +5479,55 @@ impl<'a> Lower<'a> {
             // helper; a generator body's return isn't a switch.)
             OpCode::Call => return self.call_op(pc, ins.arg as usize),
             OpCode::CallKw => return self.call_kw(pc),
+            // `await`'s and `yield from`'s delegate, through `h_await_iter`.
+            OpCode::GetAwaitable | OpCode::GetYieldFromIter if self.depth > 0 => {
+                let s = self.top_to_mem();
+                let at = self.slot_addr(s);
+                let k = self
+                    .b
+                    .ins()
+                    .iconst(types::I64, i64::from(ins.op == OpCode::GetYieldFromIter));
+                let r = self
+                    .call(h_await_iter as *const () as usize, &[self.st, at, k], true)
+                    .expect("returns");
+                let out = self.exit_with(pc, &[], INTERP);
+                self.branch_out(r, out);
+                // (A replaced instance's release may queue a finalizer.)
+                self.set_last(pc);
+                self.check_released(pc + 1);
+                true
+            }
+            // The delegate resumes as the running activation (`h_send`):
+            // the block ends either way.
+            OpCode::Send if self.depth >= 2 => {
+                self.flush();
+                self.store_last();
+                let pcv = self.b.ins().iconst(types::I64, pc as i64);
+                let len = self.b.ins().iconst(types::I64, self.depth as i64);
+                let r = self
+                    .call(h_send as *const () as usize, &[self.st, pcv, len], true)
+                    .expect("returns");
+                let out = self.exit_with(pc, &[], INTERP);
+                let switched = self.b.create_block();
+                let reload = self.b.ins().icmp_imm(IntCC::Equal, r, i64::from(RELOAD));
+                self.b.ins().brif(reload, switched, &[], out, &[]);
+                self.b.switch_to_block(switched);
+                self.b.ins().return_(&[r]);
+                return false;
+            }
+            OpCode::EndSend if self.depth >= 2 => {
+                self.flush();
+                let at = self.slot_addr(self.depth - 2);
+                let r = self
+                    .call(h_end_send as *const () as usize, &[at], true)
+                    .expect("returns");
+                let out = self.exit_with(pc, &[], INTERP);
+                self.branch_out(r, out);
+                self.depth -= 1;
+                self.set_last(pc);
+                self.check_released(pc + 1);
+                true
+            }
             OpCode::ReturnValue if !self.is_gen() && self.depth > 0 => {
                 self.flush();
                 self.store_last();
diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index e1c1e6a7..2d8c0aff 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -62,6 +62,11 @@ enum ForNext {
 /// Nested fast steps at most this deep.
 const MAX_DEPTH: u8 = 8;
 
+/// `gen_native_yield`'s stop for a release that queued a finalizer: the
+/// frame is synced for the general loop, which runs it first.
+#[cfg(feature = "jit")]
+const MARKED_AT: usize = usize::MAX - 1;
+
 /// The instructions a fast step runs (see the module docs). The scan
 /// also admits the generator prologue's `RETURN_GENERATOR` and the
 /// implicit PEP 479 handler, which a resume or only an exception reaches.
@@ -100,21 +105,54 @@ fn op_supported(op: OpCode) -> bool {
     )
 }
 
+/// The instructions a fast step leaves to the body's native form (see
+/// `frame_jit`), whose lowering of them never raises or switches in a
+/// fast step (its helpers decline without a running activation): a body
+/// whose steady state needs them takes fast steps once it's compiled.
+fn op_native(code: &CodeObject, pc: usize) -> bool {
+    let ins = code.instructions[pc];
+    match ins.op {
+        OpCode::BuildTuple => (1..=3).contains(&ins.arg),
+        OpCode::UnaryOp => ins.arg <= 3,
+        op => matches!(
+            op,
+            OpCode::LoadDeref
+                | OpCode::BinarySubscr
+                | OpCode::BinarySlice
+                | OpCode::LoadAttr
+                | OpCode::IsOp
+                | OpCode::ContainsOp
+                | OpCode::LoadGlobal
+                | OpCode::BuildList
+                | OpCode::CopyTop
+                | OpCode::Swap
+                | OpCode::PopJumpIfNone
+                | OpCode::PopJumpIfNotNone
+        ),
+    }
+}
+
+/// [`code_ok`]'s verdicts (kept in the code's extension table).
+const CODE_NO: u8 = 1;
+const CODE_OK: u8 = 2;
+/// Fast steps once the body is compiled (see [`op_native`]).
+const CODE_NATIVE: u8 = 3;
+
 /// Whether `code` is a plain generator whose steady state, everything a
 /// resume can reach from a `yield`, is instructions a fast step runs
+/// ([`CODE_OK`]), or runs once its native form does ([`CODE_NATIVE`])
 /// (cached in the code's extension table). The prologue may hold others
 /// (`range(n)` built before the loop): the first resume's step stops at
 /// them, and the general loop runs them once.
 #[inline]
-fn code_ok(code: &CodeObject) -> bool {
+fn code_ok(code: &CodeObject) -> u8 {
     use std::sync::atomic::Ordering;
     let Some(ext) = crate::code_vm_ext(code) else {
-        return false;
+        return CODE_NO;
     };
     match ext.gen_fast.load(Ordering::Relaxed) {
-        1 => false,
-        2 => true,
-        _ => code_ok_scan(code, ext),
+        0 => code_ok_scan(code, ext),
+        v => v,
     }
 }
 
@@ -122,23 +160,30 @@ fn code_ok(code: &CodeObject) -> bool {
 /// extension table.
 #[cold]
 #[inline(never)]
-fn code_ok_scan(code: &CodeObject, ext: &crate::CodeConstObjects) -> bool {
+fn code_ok_scan(code: &CodeObject, ext: &crate::CodeConstObjects) -> u8 {
     use std::sync::atomic::Ordering;
-    let ok = code.is_generator
+    let shape = code.is_generator
         && !code.is_coroutine
         && !code.is_async_generator
         && code.cellvars.is_empty()
-        && code.freevars.is_empty()
-        && in_bounds(code, ext.objects.len())
-        && steady_state_supported(code);
-    ext.gen_fast
-        .store(if ok { 2 } else { 1 }, Ordering::Relaxed);
-    ok
+        && in_bounds(code, ext.objects.len());
+    let v = if !shape {
+        CODE_NO
+    } else if code.freevars.is_empty() && steady_state_supported(code, false) {
+        CODE_OK
+    } else if cfg!(feature = "jit") && steady_state_supported(code, true) {
+        CODE_NATIVE
+    } else {
+        CODE_NO
+    };
+    ext.gen_fast.store(v, Ordering::Relaxed);
+    v
 }
 
 /// Whether every instruction a resume reaches from one of `code`'s yields
-/// is one a fast step runs (jump targets are in range: see [`in_bounds`]).
-fn steady_state_supported(code: &CodeObject) -> bool {
+/// is one a fast step runs (jump targets are in range: see [`in_bounds`]),
+/// or with `native`, one its native form runs (see [`op_native`]).
+fn steady_state_supported(code: &CodeObject, native: bool) -> bool {
     let instrs = &code.instructions;
     let n = instrs.len();
     let mut seen = vec![false; n];
@@ -151,7 +196,7 @@ fn steady_state_supported(code: &CodeObject) -> bool {
             continue;
         }
         let ins = instrs[p];
-        if !op_supported(ins.op) {
+        if !op_supported(ins.op) && !(native && op_native(code, p)) {
             return false;
         }
         let arg = ins.arg as usize;
@@ -159,7 +204,11 @@ fn steady_state_supported(code: &CodeObject) -> bool {
             OpCode::ReturnValue | OpCode::Reraise => {}
             OpCode::JumpForward => work.push(p + 1 + arg),
             OpCode::JumpBackward => work.push((p + 1).saturating_sub(arg)),
-            OpCode::PopJumpIfFalse | OpCode::PopJumpIfTrue | OpCode::ForIter => {
+            OpCode::PopJumpIfFalse
+            | OpCode::PopJumpIfTrue
+            | OpCode::PopJumpIfNone
+            | OpCode::PopJumpIfNotNone
+            | OpCode::ForIter => {
                 work.push(p + 1);
                 work.push(p + 1 + arg);
             }
@@ -189,6 +238,8 @@ fn in_bounds(code: &CodeObject, nconsts: usize) -> bool {
         match i.op {
             OpCode::PopJumpIfFalse
             | OpCode::PopJumpIfTrue
+            | OpCode::PopJumpIfNone
+            | OpCode::PopJumpIfNotNone
             | OpCode::JumpForward
             | OpCode::ForIter => p + 1 + arg < n,
             OpCode::LoadFast
@@ -336,7 +387,17 @@ impl Interpreter {
                     true
                 }
             }
-            && code_ok(&frame.code)
+            && match code_ok(&frame.code) {
+                CODE_OK => true,
+                // A body whose steady state needs its native form, compiled.
+                #[cfg(feature = "jit")]
+                CODE_NATIVE => crate::code_vm_ext(&frame.code).is_some_and(|ext| {
+                    // SAFETY: a length read with nothing running.
+                    let n = unsafe { (*frame.locals.as_ptr()).len() };
+                    ext.frame_jit.get(n).is_some()
+                }),
+                _ => false,
+            }
     }
 
     /// Run `frame` from its pc to its next `yield` (see the module docs).
@@ -458,13 +519,16 @@ impl Interpreter {
                     nst.len = len;
                     nst.pc = pc;
                     // SAFETY: the body's activation state, as this loop
-                    // holds it (its locals count checked above). A queued
-                    // finalizer waits for the general loop's next check.
-                    let _ = unsafe { native.run(nst) };
+                    // holds it (its locals count checked above).
+                    let status = unsafe { native.run(nst) };
+                    // (Its helpers never raise without a running activation.)
+                    debug_assert!(nst.err.is_none());
                     len = nst.len;
                     pc = nst.pc;
                     handed = pc;
-                    if pc >= ninstrs {
+                    // A release queued a finalizer: the general loop runs it
+                    // before the next instruction.
+                    if pc >= ninstrs || status == crate::frame_jit::MARKED {
                         break None;
                     }
                 }
@@ -740,7 +804,8 @@ impl Interpreter {
     /// `frame_jit`), without the general loop's setup: `Ok` with the value
     /// the body yielded, the frame suspended past the yield; `Err` with
     /// the pc the native code stopped at (`usize::MAX` if it didn't run),
-    /// the frame synced there for [`Self::gen_fast_step`] to go on from.
+    /// the frame synced there for [`Self::gen_fast_step`] to go on from,
+    /// or [`MARKED_AT`] for the general loop to go on from.
     #[cfg(feature = "jit")]
     #[inline]
     fn gen_native_yield(
@@ -794,7 +859,8 @@ impl Interpreter {
             gen_depth: depth,
         };
         // SAFETY: the body's activation state, as `gen_fast_step` holds it.
-        let _ = unsafe { native.run(&mut st) };
+        let status = unsafe { native.run(&mut st) };
+        debug_assert!(st.err.is_none());
         let (len, pc) = (st.len, st.pc);
         // SAFETY: the native code leaves the first `len` slots initialized.
         unsafe { frame.stack.set_len(len) };
@@ -805,6 +871,8 @@ impl Interpreter {
                 frame.agen_yielded_value = ins.arg == 0;
                 Ok(frame.stack.pop().expect("a yielded value"))
             }
+            // A release queued a finalizer: the general loop takes over.
+            _ if status == crate::frame_jit::MARKED => Err(MARKED_AT),
             _ => Err(pc),
         }
     }
@@ -850,19 +918,43 @@ impl Interpreter {
                             ForNext::Bail
                         }
                     }
-                    PyIterator::List { items, index, .. } => {
+                    PyIterator::List {
+                        items,
+                        index,
+                        owner,
+                    } => {
                         // SAFETY: as above.
-                        let Some(v) = (unsafe { items.peek() }).and_then(|xs| xs.get(*index))
-                        else {
+                        let Some(xs) = (unsafe { items.peek() }) else {
                             return ForNext::Bail;
                         };
+                        let Some(v) = xs.get(*index) else {
+                            // Exhausted: it leaves the stack (as the core
+                            // loop retires it) when its release frees
+                            // nothing; a shared one (a generator
+                            // expression's `.0`) detaches from the list, so
+                            // a later append can't resurrect it.
+                            if owner.is_some() || Rc::strong_count(items) == 1 {
+                                return ForNext::Bail;
+                            }
+                            if !unique {
+                                *items = Rc::new(crate::sync::RefCell::new(Vec::new()));
+                            }
+                            return ForNext::Exhausted;
+                        };
                         let v = copy(v);
                         *index += 1;
                         ForNext::Value(v)
                     }
                     PyIterator::Tuple { items, index } => {
                         let Some(v) = items.get(*index) else {
-                            return ForNext::Bail;
+                            // Exhausted: it leaves the stack (as the core
+                            // loop retires it) when its release can't free
+                            // the tuple, whose items could queue finalizers
+                            // (a shared iterator stays exhausted).
+                            if unique && crate::shared_value::ThinArc::strong_count(items) == 1 {
+                                return ForNext::Bail;
+                            }
+                            return ForNext::Exhausted;
                         };
                         let v = copy(v);
                         *index += 1;
@@ -969,7 +1061,15 @@ impl Interpreter {
         };
         #[cfg(not(feature = "jit"))]
         let handed = usize::MAX;
-        let out = match self.gen_fast_step(frame, snap_gen, depth, None, dead, true, handed) {
+        #[cfg(feature = "jit")]
+        let step = if handed == MARKED_AT {
+            GenStep::Bail
+        } else {
+            self.gen_fast_step(frame, snap_gen, depth, None, dead, true, handed)
+        };
+        #[cfg(not(feature = "jit"))]
+        let step = self.gen_fast_step(frame, snap_gen, depth, None, dead, true, handed);
+        let out = match step {
             GenStep::Yielded(v) => GenNext::Yielded(v),
             // The general epilogue of a return (`inline_gen_finish`): the
             // generator finishes and its frame is released (anything that
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 37d357eb..62f4bf08 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -3888,6 +3888,34 @@ impl Interpreter {
         Rc::new(RefCell::new(v))
     }
 
+    /// An empty locals vector with room for `n`, recycled when the pool
+    /// has one this frame can own (see [`Self::pooled_locals_from_args`]).
+    #[inline]
+    fn pooled_locals_empty(&self, n: usize) -> Rc<RefCell<Vec<Object>>> {
+        let pooled = {
+            let mut pool = self.frame_locals_pool.borrow_mut();
+            loop {
+                match pool.pop() {
+                    Some(rc) if Rc::strong_count(&rc) == 1 => break Some(rc),
+                    // Still shared: only our clone is dropped (no element
+                    // destructor runs under the borrow).
+                    Some(_) => (),
+                    None => break None,
+                }
+            }
+        };
+        match pooled {
+            Some(rc) => {
+                // SAFETY: sole owner, checked at the pop above.
+                let v = unsafe { &mut *rc.as_ptr() };
+                debug_assert!(v.is_empty());
+                v.reserve(n);
+                rc
+            }
+            None => Rc::new(RefCell::new(Vec::with_capacity(n))),
+        }
+    }
+
     /// RFC 0061 (WS3b): fetch a recycled argument-staging vector, or
     /// allocate one. Deliberately a separate pool from
     /// [`Self::pooled_stack`] — see the `scratch_pool` field docs.
@@ -13492,7 +13520,24 @@ impl Interpreter {
                             frame.pc = pc as u32;
                             *last_pc = last;
                             let switched = if python == 1 {
-                                self.core_call(sw, pc) || self.core_gen_call(sw, pc)
+                                if self.core_call(sw, pc) {
+                                    true
+                                } else if self.core_gen_call(sw, pc) {
+                                    // The generator, made in place: no switch,
+                                    // so the loop goes on (unless a release
+                                    // queued a finalizer).
+                                    if sw.pending.is_none() {
+                                        // SAFETY: the running activation (its
+                                        // stack only shrank: no reallocation).
+                                        len = unsafe { (*sw.cur).stack.len() };
+                                        last = pc;
+                                        pc += 1;
+                                        continue;
+                                    }
+                                    true
+                                } else {
+                                    false
+                                }
                             } else {
                                 let new = self.core_new(sw, pc, snap_gen);
                                 if new {
@@ -55972,22 +56017,34 @@ impl Interpreter {
         // CPython snapshots the *function's* current `__name__` and
         // `__qualname__` (which user code may have reassigned) into
         // `gi_name`/`gi_qualname` at call time, sharing the objects.
+        // A generator-family body's activations never start at pc 0, where
+        // the core loop counts a call's heat (see `frame_jit::Slot::warm`):
+        // its creation counts instead.
+        #[cfg(feature = "jit")]
+        if let Some(ext) = code_vm_ext(code) {
+            ext.frame_jit
+                .warm(code, ext, code.varnames.len(), frame_jit::Heat::Call);
+        }
         let gen_code = Object::Code(frame.code.clone());
-        let gen = if f.names_seeded() {
-            // The code's names, which the generator reads by default.
-            PyGenerator::new(kind, gen_code, Box::new(frame))
-        } else {
-            let (name, qualname) = f.name_objects();
-            PyGenerator::with_names(
-                name.unwrap_or_else(|| Object::from_str(f.name.clone())),
-                qualname.unwrap_or_else(|| Object::from_str(code.qualname.clone())),
-                kind,
-                gen_code,
-                Box::new(frame),
-            )
-        };
-        let gen = Rc::new(gen);
-        Self::set_frame_gen_owner(&gen);
+        let mut frame = frame;
+        // The frame's weak backlink (see `set_frame_gen_owner`), stamped as
+        // the generator is made.
+        let gen = Rc::new_cyclic(|owner| {
+            frame.gen_owner = Some(owner.clone());
+            if f.names_seeded() {
+                // The code's names, which the generator reads by default.
+                PyGenerator::new(kind, gen_code, Box::new(frame))
+            } else {
+                let (name, qualname) = f.name_objects();
+                PyGenerator::with_names(
+                    name.unwrap_or_else(|| Object::from_str(f.name.clone())),
+                    qualname.unwrap_or_else(|| Object::from_str(code.qualname.clone())),
+                    kind,
+                    gen_code,
+                    Box::new(frame),
+                )
+            }
+        });
         let obj = if code.is_coroutine {
             Object::Coroutine(gen)
         } else if code.is_async_generator {
@@ -56064,11 +56121,29 @@ impl Interpreter {
         if missing > 0 && (missing > f.defaults.len() || f.defaults_maybe_overridden()) {
             return None;
         }
+        let ext = code_vm_ext(code)?;
+        match ext.gen_start.load(std::sync::atomic::Ordering::Relaxed) {
+            0 => Self::generator_start_scan(code, ext),
+            u32::MAX => None,
+            at => Some(at as usize - 1),
+        }
+    }
+
+    /// [`Self::generator_start`]'s scan of `code`'s prologue, remembered
+    /// in its extension table.
+    #[cold]
+    #[inline(never)]
+    fn generator_start_scan(code: &CodeObject, ext: &CodeConstObjects) -> Option<usize> {
         let start = code
             .instructions
             .iter()
-            .position(|i| !matches!(i.op, OpCode::MakeCell | OpCode::CopyFreeVars))?;
-        (code.instructions[start].op == OpCode::ReturnGenerator).then_some(start)
+            .position(|i| !matches!(i.op, OpCode::MakeCell | OpCode::CopyFreeVars))
+            .filter(|&start| code.instructions[start].op == OpCode::ReturnGenerator);
+        ext.gen_start.store(
+            start.map_or(u32::MAX, |at| at as u32 + 1),
+            std::sync::atomic::Ordering::Relaxed,
+        );
+        start
     }
 
     /// [`Self::start_generator_fast`] once [`Self::generator_start`] has
@@ -56084,17 +56159,63 @@ impl Interpreter {
         if missing > 0 {
             positional.extend(f.defaults[f.defaults.len() - missing..].iter().cloned());
         }
-        let mut frame = self.make_frame(
-            code.clone(),
-            positional,
-            f.closure.clone(),
-            f.globals.clone(),
-            Some(f.builtins.clone()),
-        );
-        frame.pc = start as u32 + 1;
+        let frame = match f.lean_cells_ref(code) {
+            // No cells of its own: the function's (see `core_gen_call`).
+            Some(cells) => {
+                let locals = self.pooled_locals_from_args(&mut positional, code.varnames.len());
+                self.recycle_scratch(positional);
+                let mut frame = Self::gen_frame_lean(f, code.clone(), locals, cells.clone(), start);
+                frame.stack = self.pooled_stack();
+                frame
+            }
+            None => {
+                let mut frame = self.make_frame(
+                    code.clone(),
+                    positional,
+                    f.closure.clone(),
+                    f.globals.clone(),
+                    Some(f.builtins.clone()),
+                );
+                frame.pc = start as u32 + 1;
+                frame
+            }
+        };
         Self::wrap_started_generator(f, code, frame)
     }
 
+    /// The frame of a generator-family call of `f` (its code `code`, which
+    /// defines no cells of its own) with its `locals` filled and `cells`
+    /// the function's, just past the `RETURN_GENERATOR` at `start`.
+    #[inline]
+    fn gen_frame_lean(
+        f: &PyFunction,
+        code: Rc<CodeObject>,
+        locals: Rc<RefCell<Vec<Object>>>,
+        cells: Rc<Vec<Rc<RefCell<Object>>>>,
+        start: usize,
+    ) -> Frame {
+        Frame {
+            code,
+            locals,
+            cells,
+            stack: Vec::new(),
+            globals: f.globals.clone(),
+            builtins: f.builtins.clone(),
+            rare: None,
+            exc: None,
+            agen_yielded_value: true,
+            pc: start as u32 + 1,
+            py_frame: None,
+            gen_owner: None,
+            suppress_call_event: false,
+            gen_first_resume: false,
+            sent_consumed: false,
+            shell_cache: None,
+            #[cfg(feature = "jit")]
+            parked_native: None,
+        }
+    }
+
     /// The core loop's `CALL` at `pc` of a generator function (a plain
     /// function, or a method over one), in `sw`'s synced running
     /// activation: the generator, made in place (see
@@ -56122,34 +56243,58 @@ impl Interpreter {
             }
             _ => return false,
         };
-        let code = f.code();
-        let Some(start) = self.generator_start(f, &code, argc + usize::from(receiver.is_some()))
+        // SAFETY: GIL-serialized raw read of the function's code cell (the
+        // callee on the stack holds the function).
+        let code_ref: &Rc<CodeObject> = unsafe { &*f.code.as_ptr() };
+        let Some(start) =
+            self.generator_start(f, code_ref, argc + usize::from(receiver.is_some()))
         else {
             return false;
         };
+        let code = code_ref.clone();
+        // A body without cells of its own shares the function's (see
+        // `PyFunction::lean_cells`), and its frame is built right here.
+        let cells = f.lean_cells_ref(&code).cloned();
         // Committed: the arguments move into the generator's frame, and
         // the callee and its self slot leave the stack (one that dies
         // queues its finalizer, which runs before the next instruction).
-        let mut positional = self.pooled_scratch();
         let mut ops = frame.stack.drain(callee_at..);
         let callee = ops.next().expect("checked above");
         let slot = ops.next().expect("checked above");
-        let callee_receiver = match &callee {
-            Object::BoundMethod(bm) => Some(bm.receiver.clone()),
-            _ => None,
+        let (f, callee_receiver) = match &callee {
+            Object::Function(f) => (f, None),
+            Object::BoundMethod(bm) => match &bm.function {
+                Object::Function(f) => (f, Some(bm.receiver.clone())),
+                _ => unreachable!("checked above"),
+            },
+            _ => unreachable!("checked above"),
         };
-        positional.extend(callee_receiver);
-        positional.extend((!matches!(slot, Object::Unbound)).then_some(slot));
-        positional.extend(ops);
-        let gen = match &callee {
-            Object::Function(f) => self.start_generator_at(f, &code, positional, start),
-            Object::BoundMethod(bm) => {
-                let Object::Function(f) = &bm.function else {
-                    unreachable!("checked above");
-                };
-                self.start_generator_at(f, &code, positional, start)
+        let gen = if let Some(cells) = cells {
+            // The arguments move straight into the locals (no staging).
+            let nlocals = code.varnames.len();
+            let locals = self.pooled_locals_empty(nlocals);
+            {
+                // SAFETY: sole owner (see `pooled_locals_empty`).
+                let v = unsafe { &mut *locals.as_ptr() };
+                v.extend(callee_receiver);
+                v.extend((!matches!(slot, Object::Unbound)).then_some(slot));
+                v.extend(ops);
+                let missing = (code.arg_count as usize).saturating_sub(v.len());
+                if missing > 0 {
+                    v.extend(f.defaults[f.defaults.len() - missing..].iter().cloned());
+                }
+                v.resize(nlocals, Object::Unbound);
             }
-            _ => unreachable!("checked above"),
+            let mut gen_frame = Self::gen_frame_lean(f, code, locals, cells, start);
+            gen_frame.stack = self.pooled_stack();
+            let code = Rc::clone(&gen_frame.code);
+            Self::wrap_started_generator(f, &code, gen_frame)
+        } else {
+            let mut positional = self.pooled_scratch();
+            positional.extend(callee_receiver);
+            positional.extend((!matches!(slot, Object::Unbound)).then_some(slot));
+            positional.extend(ops);
+            self.start_generator_at(f, &code, positional, start)
         };
         self.release(callee);
         frame.stack.push(gen);
@@ -70603,6 +70748,11 @@ struct CodeConstObjects {
     /// Whether the code is a generator body fast steps run (see
     /// `gen_fast`): 0 not yet scanned, 1 no, 2 yes.
     gen_fast: std::sync::atomic::AtomicU8,
+    /// The `RETURN_GENERATOR` a generator-family call of the code starts
+    /// past (see `Interpreter::generator_start`): `0` not yet scanned,
+    /// `u32::MAX` none (the prologue holds something else), else its pc
+    /// plus one.
+    gen_start: std::sync::atomic::AtomicU32,
     /// Split-layout attribute shortcuts per `LOAD_ATTR` site (see
     /// [`FieldSlot`]); allocated on the first recorded one.
     field_slots: std::sync::OnceLock<Box<[FieldSlot]>>,
@@ -72718,6 +72868,7 @@ fn code_vm_ext_build(code: &CodeObject) -> &CodeConstObjects {
             leaf_tries: std::sync::atomic::AtomicU64::new(0),
             fast_pairs: std::sync::OnceLock::new(),
             gen_fast: std::sync::atomic::AtomicU8::new(0),
+            gen_start: std::sync::atomic::AtomicU32::new(0),
             field_slots: std::sync::OnceLock::new(),
             native_sites: std::sync::OnceLock::new(),
             instance_sites: std::sync::OnceLock::new(),
diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs
index b3b56f99..466b10bc 100644
--- a/crates/weavepy-vm/src/stdlib/sys.rs
+++ b/crates/weavepy-vm/src/stdlib/sys.rs
@@ -2826,9 +2826,18 @@ thread_local! {
     static CORO_ORIGIN_DEPTH: std::cell::Cell<i64> = const { std::cell::Cell::new(0) };
 }
 
+/// Whether any thread ever set a nonzero coroutine origin tracking depth:
+/// until one does, every thread's depth is `0` without a thread-local read
+/// (the coroutine constructors ask on every call).
+static CORO_ORIGIN_EVER: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
 /// Current `sys.get_coroutine_origin_tracking_depth()` value; read by
 /// the interpreter when constructing coroutine objects.
+#[inline]
 pub fn coroutine_origin_tracking_depth() -> i64 {
+    if !CORO_ORIGIN_EVER.load(std::sync::atomic::Ordering::Relaxed) {
+        return 0;
+    }
     CORO_ORIGIN_DEPTH.with(std::cell::Cell::get)
 }
 
@@ -2845,6 +2854,9 @@ fn sys_set_coroutine_origin_tracking_depth(args: &[Object]) -> Result<Object, Ru
     if depth < 0 {
         return Err(crate::error::value_error("depth must be >= 0"));
     }
+    if depth != 0 {
+        CORO_ORIGIN_EVER.store(true, std::sync::atomic::Ordering::Relaxed);
+    }
     CORO_ORIGIN_DEPTH.with(|c| c.set(depth));
     Ok(Object::None)
 }
diff --git a/tests/regrtest/test_await_native_bodies.py b/tests/regrtest/test_await_native_bodies.py
new file mode 100644
index 00000000..e2050945
--- /dev/null
+++ b/tests/regrtest/test_await_native_bodies.py
@@ -0,0 +1,194 @@
+"""Coroutine and `yield from` bodies once they're compiled: the native code
+runs `GET_AWAITABLE`, `GET_YIELD_FROM_ITER`, `SEND` and `END_SEND` (see
+`frame_jit` in the VM).
+
+Each case warms a body until it compiles, then checks a behavior at the
+boundary between the native code and the core loop: results, raises,
+suspensions, `throw`/`close`, introspection of the delegate, and errors
+for bad awaitables.
+"""
+
+import types
+import unittest
+
+WARM = 30000
+
+
+def drive(coro):
+    try:
+        while True:
+            coro.send(None)
+    except StopIteration as e:
+        return e.value
+
+
+async def fib(n):
+    if n <= 1:
+        return n
+    return await fib(n - 1) + await fib(n - 2)
+
+
+class Suspend:
+    def __await__(self):
+        value = yield "suspended"
+        return value
+
+
+async def waiter(n):
+    total = 0
+    for i in range(n):
+        total += await Suspend() or i
+    return total
+
+
+async def leaf(x):
+    if x < 0:
+        raise ValueError(x)
+    return x * 2
+
+
+async def caller(x):
+    return await leaf(x) + 1
+
+
+class Tree:
+    __slots__ = ("left", "value", "right")
+
+    def __init__(self, left, value, right):
+        self.left = left
+        self.value = value
+        self.right = right
+
+    def __iter__(self):
+        if self.left:
+            yield from self.left
+        yield self.value
+        if self.right:
+            yield from self.right
+
+
+def build(lo, hi):
+    if lo >= hi:
+        return None
+    mid = (lo + hi) // 2
+    return Tree(build(lo, mid), mid, build(mid + 1, hi))
+
+
+def delegator(it):
+    result = yield from it
+    return result
+
+
+def returner(n):
+    yield n
+    return n * 10
+
+
+class AwaitNativeTests(unittest.TestCase):
+    def test_fib(self):
+        for _ in range(3):
+            drive(fib(15))
+        self.assertEqual(drive(fib(20)), 6765)
+
+    def test_suspend_and_send(self):
+        for _ in range(WARM // 100):
+            drive(waiter(100))
+        coro = waiter(3)
+        self.assertEqual(coro.send(None), "suspended")
+        self.assertIsNotNone(coro.cr_await)
+        self.assertEqual(coro.send(10), "suspended")
+        self.assertEqual(coro.send(None), "suspended")
+        with self.assertRaises(StopIteration) as cm:
+            coro.send(None)
+        self.assertEqual(cm.exception.value, 10 + 1 + 2)
+        self.assertIsNone(coro.cr_await)
+
+    def test_raise_through_await(self):
+        for i in range(WARM):
+            drive(caller(i))
+        try:
+            drive(caller(-5))
+        except ValueError as e:
+            exc = e
+        else:
+            self.fail("no ValueError")
+        self.assertEqual(exc.args, (-5,))
+        names = []
+        tb = exc.__traceback__
+        while tb is not None:
+            names.append(tb.tb_frame.f_code.co_name)
+            tb = tb.tb_next
+        self.assertEqual(names[-2:], ["caller", "leaf"])
+
+    def test_throw_and_close(self):
+        for _ in range(WARM // 100):
+            drive(waiter(100))
+        coro = waiter(5)
+        coro.send(None)
+        with self.assertRaises(KeyError):
+            coro.throw(KeyError("x"))
+        coro = waiter(5)
+        coro.send(None)
+        coro.close()
+        self.assertIsNone(coro.cr_frame)
+
+    def test_bad_awaitables(self):
+        async def bad():
+            return await 5
+
+        async def twice():
+            c = leaf(1)
+            gen = c.__await__()
+            next(gen, None)
+            return await c
+
+        for i in range(WARM):
+            drive(caller(i))
+        with self.assertRaises(TypeError):
+            drive(bad())
+        with self.assertRaises(RuntimeError):
+            drive(twice())
+
+    def test_types_coroutine(self):
+        @types.coroutine
+        def legacy():
+            value = yield "legacy"
+            return value
+
+        async def uses(n):
+            return await legacy() + n
+
+        for i in range(WARM // 10):
+            c = uses(i)
+            c.send(None)
+            try:
+                c.send(1)
+            except StopIteration as e:
+                self.assertEqual(e.value, i + 1)
+
+    def test_yield_from_tree(self):
+        tree = build(0, 200)
+        for _ in range(200):
+            self.assertEqual(sum(tree), sum(range(200)))
+        self.assertEqual(list(tree), list(range(200)))
+        it = iter(tree)
+        self.assertEqual(next(it), 0)
+        self.assertIsNotNone(it.gi_yieldfrom)
+        self.assertEqual(next(it), 1)
+        it.close()
+        self.assertIsNone(it.gi_frame)
+
+    def test_yield_from_return_value(self):
+        for i in range(WARM):
+            list(delegator(returner(i)))
+        gen = delegator(returner(4))
+        self.assertEqual(next(gen), 4)
+        with self.assertRaises(StopIteration) as cm:
+            next(gen)
+        self.assertEqual(cm.exception.value, 40)
+        gen = delegator(iter([1, 2]))
+        self.assertEqual(list(gen), [1, 2])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/regrtest/test_generator_closure_fast_steps.py b/tests/regrtest/test_generator_closure_fast_steps.py
new file mode 100644
index 00000000..7f260210
--- /dev/null
+++ b/tests/regrtest/test_generator_closure_fast_steps.py
@@ -0,0 +1,156 @@
+"""Generator expressions over closures and subscripts once their bodies
+are compiled: fast steps run them natively (see `gen_fast` in the VM).
+
+Each case warms a generator expression until its body compiles, then
+checks a behavior at the boundary between the native steps and the
+general loop: raises, unbound closure cells, exhausted iterators, and
+finalizers of objects an exhausted iterator releases.
+"""
+
+import unittest
+
+WARM = 3000
+
+
+class Tracked:
+    deleted = 0
+
+    def __del__(self):
+        Tracked.deleted += 1
+
+
+def warm(fn, *args):
+    for _ in range(WARM):
+        fn(*args)
+
+
+def tuple_of(pool, indices):
+    return tuple(pool[i] for i in indices)
+
+
+def set_of(vec, cols):
+    return set(vec[i] + i for i in cols)
+
+
+def sum_of(vec, cols):
+    return sum(vec[i] * 2 for i in cols)
+
+
+def attr_count(items, k):
+    return sum(1 for w in items if w.kind is k)
+
+
+class Item:
+    __slots__ = ("kind",)
+
+    def __init__(self, kind):
+        self.kind = kind
+
+
+class ClosureFastStepTests(unittest.TestCase):
+    def test_results(self):
+        pool = (10, 11, 12, 13)
+        warm(tuple_of, pool, [3, 2, 1, 0])
+        self.assertEqual(tuple_of(pool, [3, 2, 1, 0]), (13, 12, 11, 10))
+        self.assertEqual(tuple_of(pool, []), ())
+        vec = (3, 1, 0, 2)
+        warm(set_of, vec, range(4))
+        self.assertEqual(set_of(vec, range(4)), {3, 2, 2, 5})
+        warm(sum_of, vec, range(4))
+        self.assertEqual(sum_of(vec, range(4)), 12)
+        items = [Item(i % 3) for i in range(10)]
+        warm(attr_count, items, 1)
+        self.assertEqual([attr_count(items, k) for k in (0, 1, 2)], [4, 3, 3])
+
+    def test_raise_after_warm_up(self):
+        pool = (1, 2, 3)
+        warm(tuple_of, pool, [0, 1, 2])
+        with self.assertRaises(IndexError):
+            tuple_of(pool, [0, 1, 7])
+        gen = (pool[i] for i in [0, 9])
+        self.assertEqual(next(gen), 1)
+        with self.assertRaises(IndexError):
+            next(gen)
+        # The generator finished with the raise.
+        self.assertEqual(list(gen), [])
+        with self.assertRaises(TypeError):
+            tuple_of(pool, [0, "x"])
+
+    def test_unbound_cell(self):
+        def make():
+            v = (1, 2, 3)
+            gen = (v[i] for i in range(3))
+            first = next(gen)
+            del v
+            return first, gen
+
+        for _ in range(WARM):
+            v = (1, 2)
+            list(v[i] for i in range(2))
+        first, gen = make()
+        self.assertEqual(first, 1)
+        with self.assertRaises(NameError):
+            next(gen)
+
+    def test_shared_tuple_iterator(self):
+        def drain(it):
+            return list(x + 1 for x in it)
+
+        for _ in range(WARM):
+            drain(iter((1, 2, 3)))
+        it = iter((1, 2, 3))
+        self.assertEqual(drain(it), [2, 3, 4])
+        self.assertEqual(next(it, "end"), "end")
+        self.assertEqual(list(it), [])
+
+    def test_shared_list_iterator_detaches(self):
+        def drain(it):
+            return list(x * 2 for x in it)
+
+        for _ in range(WARM):
+            drain(iter([1, 2]))
+        lst = [1, 2, 3]
+        it = iter(lst)
+        self.assertEqual(drain(it), [2, 4, 6])
+        lst.append(4)
+        self.assertEqual(next(it, "end"), "end")
+        # A generator expression's own `.0` over a list that grows later.
+        lst = [5, 6]
+        gen = (x for x in lst)
+        self.assertEqual(list(gen), [5, 6])
+        lst.append(7)
+        self.assertEqual(list(gen), [])
+
+    def test_exhausted_tuple_releases_items(self):
+        def count(a, b):
+            return sum(1 for x in (a, b))
+
+        for _ in range(WARM):
+            count(1, 2)
+        Tracked.deleted = 0
+        n = count(Tracked(), Tracked())
+        # The iterator's tuple held the last references.
+        self.assertEqual(n, 2)
+        self.assertEqual(Tracked.deleted, 2)
+
+        def first_only(a, b):
+            for x in (y for y in (a, b)):
+                return x
+
+        Tracked.deleted = 0
+        for _ in range(WARM):
+            first_only(1, 2)
+        first_only(Tracked(), Tracked())
+        self.assertEqual(Tracked.deleted, 2)
+
+    def test_nested_closure_generators(self):
+        def outer(xs, k):
+            inner = (x * k for x in xs)
+            return [y + k for y in (v for v in inner)]
+
+        warm(outer, (1, 2, 3), 2)
+        self.assertEqual(outer((1, 2, 3), 2), [4, 6, 8])
+
+
+if __name__ == "__main__":
+    unittest.main()

From 2fb29e731e38a613531cfefd73834914a979b5c7 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 18:57:41 -0700
Subject: [PATCH 011/173] perf: send into generators without a bound method

`gen.send(v)` as a method call loads the unbound `send` (the `.u.`
sentinel `type(gen).send` gives) with the generator as its self, so no
bound method is made and released per send; the core loop's `CALL` and
the frame JIT's call helper both resume the generator inline for either
form. `send` now checks its one argument as CPython's `METH_O` does
(`generator.send() takes exactly one argument (0 given)`), which the
bound form didn't before.

Generator creation reads the code's shape (a generator-family body with
positional parameters only) from the cached `RETURN_GENERATOR` verdict
instead of testing it per call, and a new generator joins its young set
directly (`gc_trace::track_generator`).

Instructions retired vs the branch point (suite, per call): coroutines
0.883, nqueens 0.881, generators_tree 0.946, geomean 0.991. Probes:
`avg.send(i)` 1.55x -> 1.36x CPython, an `await` of an `__await__`
generator 2.12x -> 2.00x.
---
 crates/weavepy-vm/src/frame_jit.rs           |  13 ++
 crates/weavepy-vm/src/gc_trace.rs            |  14 ++
 crates/weavepy-vm/src/lib.rs                 | 141 ++++++++++++++----
 tests/regrtest/test_generator_send_method.py | 142 +++++++++++++++++++
 4 files changed, 280 insertions(+), 30 deletions(-)
 create mode 100644 tests/regrtest/test_generator_send_method.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 3bb84e68..3dae9a90 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3112,6 +3112,19 @@ unsafe extern "C" fn h_call(
             frame.pc = pc as u32;
             *(*st.sw).last = st.last;
         };
+        // `gen.send(v)`: likewise, with the value sent in.
+        if argc == 1
+            && !st.sw.is_null()
+            && Interpreter::gen_send_call(&*base.add(start), &*base.add(start + 1))
+        {
+            sync(st);
+            let sw = &mut *st.sw;
+            let interp = &mut *st.interp.cast_mut();
+            if !interp.core_gen_resume(sw, pc, crate::InlineResume::SendCall) {
+                sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+            }
+            return RELOAD;
+        }
         // `next(gen)`: the generator resumes inline, as for `FOR_ITER`,
         // with the yield as the call's result.
         if argc == 1
diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs
index 4c4ef241..7462ce37 100644
--- a/crates/weavepy-vm/src/gc_trace.rs
+++ b/crates/weavepy-vm/src/gc_trace.rs
@@ -2659,6 +2659,20 @@ pub fn track(obj: &Object) {
     with_state(|s| s.track(obj));
 }
 
+/// [`track`] for a fresh generator, coroutine or async generator: it joins
+/// its young set (see [`GcState::nurse`]), or is registered at once.
+pub fn track_generator(obj: &Object) {
+    let (Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g)) = obj else {
+        track(obj);
+        return;
+    };
+    with_state(|s| {
+        if !s.nurse(&s.young_gens, g) {
+            s.track_now(obj);
+        }
+    });
+}
+
 /// Track a list or dict that native code filled before publishing it (the
 /// unpickler's results) as if it had been tracked at birth, while still
 /// empty: it joins the deferred containers. Every collection promotes the
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 62f4bf08..e12116fc 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -8292,6 +8292,14 @@ impl Interpreter {
                         _ => return None,
                     }
                 }
+                // The method form `gen.send` loads (see `core_gen_method`).
+                (
+                    Some(Object::Builtin(b)),
+                    Some(Object::Generator(g) | Object::Coroutine(g)),
+                    Some(v),
+                ) if matches!(b.name, ".u.gen_send" | ".u.cor_send") => {
+                    (g, matches!(v, Object::None))
+                }
                 _ => return None,
             }
         } else if send {
@@ -8374,13 +8382,14 @@ impl Interpreter {
         match mode {
             // `next`, its empty self slot and the generator (held above).
             InlineResume::NextCall => {
-                let n = frame.stack.len();
-                drop(frame.stack.drain(n - 3..));
+                let (a, b, c) = (frame.stack.pop(), frame.stack.pop(), frame.stack.pop());
+                drop((a, b, c));
             }
-            // The bound `send` and its empty self slot (the value went in).
+            // The bound `send` and its empty self slot, or the method and
+            // the generator (the value went in).
             InlineResume::SendCall => {
-                let n = frame.stack.len();
-                drop(frame.stack.drain(n - 2..));
+                let (a, b) = (frame.stack.pop(), frame.stack.pop());
+                drop((a, b));
             }
             _ => {}
         }
@@ -13431,12 +13440,9 @@ impl Interpreter {
                         // SAFETY: `len >= argc + 2 == 3`.
                         if argc == 1
                             && !plain_fn
-                            && matches!(
-                                unsafe { (&*base.add(len - 3), &*base.add(len - 2)) },
-                                (Object::BoundMethod(bm), Object::Unbound)
-                                    if matches!(&bm.function, Object::Builtin(b)
-                                        if matches!(b.name, ".gen_send" | ".cor_send"))
-                            )
+                            && Self::gen_send_call(unsafe { &*base.add(len - 3) }, unsafe {
+                                &*base.add(len - 2)
+                            })
                         {
                             // SAFETY: `len <= cap`, every slot initialized.
                             unsafe { frame.stack.set_len(len) };
@@ -49861,10 +49867,13 @@ impl Interpreter {
                         })?;
                         let rest: &[Object] = if args.is_empty() { &[] } else { &args[1..] };
                         return match b.name {
-                            ".u.gen_send" | ".u.cor_send" => self.gen_method_send(
-                                &receiver,
-                                rest.first().cloned().unwrap_or(Object::None),
-                            ),
+                            ".u.gen_send" | ".u.cor_send" => {
+                                gen_send_arity(&receiver, rest.len())?;
+                                self.gen_method_send(
+                                    &receiver,
+                                    rest.first().cloned().unwrap_or(Object::None),
+                                )
+                            }
                             ".u.gen_throw" | ".u.cor_throw" => {
                                 self.gen_method_throw(&receiver, rest)
                             }
@@ -51090,6 +51099,7 @@ impl Interpreter {
                             return Err(type_error("__subclasses__() requires a type receiver"));
                         }
                         ".gen_send" | ".cor_send" => {
+                            gen_send_arity(&bm.receiver, args.len())?;
                             let value = args.first().cloned().unwrap_or(Object::None);
                             return self.gen_method_send(&bm.receiver, value);
                         }
@@ -56055,7 +56065,7 @@ impl Interpreter {
         // RFC 0024: generator frames can participate in reference cycles
         // (a local that holds the generator itself), so track them like
         // instances.
-        gc_trace::track(&obj);
+        gc_trace::track_generator(&obj);
         obj
     }
 
@@ -56104,12 +56114,14 @@ impl Interpreter {
     /// call of `f` (its code `code`) with `given` positional arguments:
     /// the `RETURN_GENERATOR` it starts past, if so.
     fn generator_start(&self, f: &PyFunction, code: &CodeObject, given: usize) -> Option<usize> {
+        let ext = code_vm_ext(code)?;
+        let start = match ext.gen_start.load(std::sync::atomic::Ordering::Relaxed) {
+            0 => Self::generator_start_scan(code, ext)?,
+            u32::MAX => return None,
+            at => at as usize - 1,
+        };
         let nargs = code.arg_count as usize;
-        if !(code.is_generator || code.is_coroutine || code.is_async_generator)
-            || code.kwonly_count != 0
-            || code.has_varargs
-            || code.has_varkeywords
-            || given > nargs
+        if given > nargs
             || (code.is_coroutine && crate::stdlib::sys::coroutine_origin_tracking_depth() > 0)
             || crate::trace::eval_frame_record_active()
             || crate::trace::any_observers_active()
@@ -56121,24 +56133,25 @@ impl Interpreter {
         if missing > 0 && (missing > f.defaults.len() || f.defaults_maybe_overridden()) {
             return None;
         }
-        let ext = code_vm_ext(code)?;
-        match ext.gen_start.load(std::sync::atomic::Ordering::Relaxed) {
-            0 => Self::generator_start_scan(code, ext),
-            u32::MAX => None,
-            at => Some(at as usize - 1),
-        }
+        Some(start)
     }
 
-    /// [`Self::generator_start`]'s scan of `code`'s prologue, remembered
-    /// in its extension table.
+    /// [`Self::generator_start`]'s verdict on `code`'s shape and prologue,
+    /// remembered in its extension table: a generator-family body with
+    /// positional parameters only, and the `RETURN_GENERATOR` it starts
+    /// past.
     #[cold]
     #[inline(never)]
     fn generator_start_scan(code: &CodeObject, ext: &CodeConstObjects) -> Option<usize> {
+        let shaped = (code.is_generator || code.is_coroutine || code.is_async_generator)
+            && code.kwonly_count == 0
+            && !code.has_varargs
+            && !code.has_varkeywords;
         let start = code
             .instructions
             .iter()
             .position(|i| !matches!(i.op, OpCode::MakeCell | OpCode::CopyFreeVars))
-            .filter(|&start| code.instructions[start].op == OpCode::ReturnGenerator);
+            .filter(|&start| shaped && code.instructions[start].op == OpCode::ReturnGenerator);
         ext.gen_start.store(
             start.map_or(u32::MAX, |at| at as u32 + 1),
             std::sync::atomic::Ordering::Relaxed,
@@ -56216,6 +56229,23 @@ impl Interpreter {
         }
     }
 
+    /// Whether a `CALL` of one argument with `callee` and `slot` below it
+    /// is `gen.send(v)` or `coro.send(v)`: the bound method with an empty
+    /// self slot, or the method form `gen.send` loads (see
+    /// `core_gen_method`).
+    #[inline(always)]
+    pub(crate) fn gen_send_call(callee: &Object, slot: &Object) -> bool {
+        match (callee, slot) {
+            (Object::BoundMethod(bm), Object::Unbound) => {
+                matches!(&bm.function, Object::Builtin(b) if matches!(b.name, ".gen_send" | ".cor_send"))
+            }
+            (Object::Builtin(b), Object::Generator(_) | Object::Coroutine(_)) => {
+                matches!(b.name, ".u.gen_send" | ".u.cor_send")
+            }
+            _ => false,
+        }
+    }
+
     /// The core loop's `CALL` at `pc` of a generator function (a plain
     /// function, or a method over one), in `sw`'s synced running
     /// activation: the generator, made in place (see
@@ -66710,6 +66740,20 @@ unsafe fn core_gen_method(code: &CodeObject, name_idx: u32, top: *mut Object) ->
     else {
         return false;
     };
+    // `gen.send(v)`: the unbound method and the generator as its self (the
+    // `CALL` passes it as the first argument), with no bound method made;
+    // the `CALL` arms resume the generator inline.
+    if name == "send" {
+        // SAFETY: the caller's contract.
+        let coro = matches!(unsafe { &*top }, Object::Coroutine(_));
+        // SAFETY: the generator moves up to the self slot, which is free.
+        unsafe {
+            let gen = top.read();
+            top.write(gen_send_unbound(coro));
+            top.add(1).write(gen);
+        }
+        return true;
+    }
     // SAFETY: the caller's contract.
     let bm = make_gen_method(name, unsafe { &*top });
     // SAFETY: the generator (held by the bound method too) is replaced in
@@ -66721,6 +66765,43 @@ unsafe fn core_gen_method(code: &CodeObject, name_idx: u32, top: *mut Object) ->
     true
 }
 
+/// `send`'s one argument (CPython's `METH_O`): `n` given to `receiver`'s.
+fn gen_send_arity(receiver: &Object, n: usize) -> Result<(), RuntimeError> {
+    if n == 1 {
+        return Ok(());
+    }
+    let kind = match receiver {
+        Object::Coroutine(_) => "coroutine",
+        _ => "generator",
+    };
+    Err(type_error(format!(
+        "{kind}.send() takes exactly one argument ({n} given)"
+    )))
+}
+
+/// The unbound `send` of generators (or coroutines) that a method load of
+/// `gen.send` pushes (see [`core_gen_method`]): the `.u.` sentinel a
+/// type's attribute gives (`type(gen).send`), which `Interpreter::call`
+/// runs with its first argument as the receiver. One per kind, shared.
+fn gen_send_unbound(coro: bool) -> Object {
+    static METHODS: [std::sync::OnceLock<Object>; 2] = [const { std::sync::OnceLock::new() }; 2];
+    METHODS[usize::from(coro)]
+        .get_or_init(|| {
+            Object::Builtin(Rc::new(BuiltinFn {
+                name: if coro { ".u.cor_send" } else { ".u.gen_send" },
+                binds_instance: false,
+                call: Box::new(|_args| {
+                    Err(RuntimeError::Internal(
+                        "unbound generator method must be dispatched via Interpreter::call"
+                            .to_owned(),
+                    ))
+                }),
+                call_kw: None,
+            }))
+        })
+        .clone()
+}
+
 fn make_gen_method(name: &str, receiver: &Object) -> Object {
     fn unreachable_call(_args: &[Object]) -> Result<Object, RuntimeError> {
         Err(RuntimeError::Internal(
diff --git a/tests/regrtest/test_generator_send_method.py b/tests/regrtest/test_generator_send_method.py
new file mode 100644
index 00000000..7f8c3d4c
--- /dev/null
+++ b/tests/regrtest/test_generator_send_method.py
@@ -0,0 +1,142 @@
+"""`gen.send(v)` and `coro.send(v)` as method calls: the method load
+pushes the unbound `send` with the generator as its self, and the `CALL`
+resumes the generator inline (see `core_gen_method` in the VM).
+
+Each case runs warm (in loops long enough to compile) and checks the
+general path's behaviors too: arity errors, sends into fresh, running
+and finished generators, frames made visible, and the bound and unbound
+forms of the method.
+"""
+
+import unittest
+
+WARM = 3000
+
+
+def averager():
+    total = 0.0
+    count = 0
+    avg = None
+    while True:
+        value = yield avg
+        total += value
+        count += 1
+        avg = total / count
+
+
+def echo():
+    value = None
+    while True:
+        value = yield value
+
+
+class SendMethodTests(unittest.TestCase):
+    def test_warm_sends(self):
+        avg = averager()
+        next(avg)
+        last = None
+        for i in range(WARM):
+            last = avg.send(i)
+        self.assertEqual(last, (WARM - 1) / 2)
+
+    def test_arity(self):
+        g = echo()
+        next(g)
+        for _ in range(WARM):
+            g.send(1)
+        with self.assertRaises(TypeError):
+            g.send()
+        with self.assertRaises(TypeError):
+            g.send(1, 2)
+        self.assertEqual(g.send(3), 3)
+
+    def test_fresh_running_finished(self):
+        for _ in range(WARM):
+            g = echo()
+            g.send(None)
+        g = echo()
+        with self.assertRaises(TypeError):
+            g.send(1)
+        self.assertIsNone(g.send(None))
+
+        def selfsend():
+            yield me.send(None)
+
+        me = selfsend()
+        with self.assertRaises(ValueError):
+            next(me)
+
+        def short():
+            yield 1
+
+        g = short()
+        g.send(None)
+        with self.assertRaises(StopIteration):
+            g.send(None)
+        with self.assertRaises(StopIteration):
+            g.send(None)
+
+    def test_visible_frame(self):
+        g = echo()
+        next(g)
+        for i in range(WARM):
+            self.assertEqual(g.send(i), i)
+        self.assertEqual(g.gi_frame.f_locals["value"], WARM - 1)
+        for i in range(10):
+            self.assertEqual(g.send(i), i)
+        self.assertFalse(g.gi_running)
+
+    def test_bound_and_unbound_forms(self):
+        g = echo()
+        next(g)
+        send = g.send
+        for i in range(WARM):
+            self.assertEqual(send(i), i)
+        self.assertEqual(type(g).send(g, 7), 7)
+        self.assertEqual(getattr(g, "send")(8), 8)
+        with self.assertRaises(TypeError):
+            type(g).send(1, 2)
+
+    def test_coroutine_send(self):
+        class Suspend:
+            def __await__(self):
+                return (yield "s")
+
+        async def coro():
+            total = 0
+            while True:
+                total += await Suspend()
+                if total > 10 ** 9:
+                    return total
+
+        c = coro()
+        self.assertEqual(c.send(None), "s")
+        for i in range(WARM):
+            self.assertEqual(c.send(i), "s")
+        with self.assertRaises(StopIteration) as cm:
+            c.send(10 ** 10)
+        self.assertEqual(cm.exception.value, sum(range(WARM)) + 10 ** 10)
+        c2 = coro()
+        with self.assertRaises(TypeError):
+            c2.send(1)
+        c2.close()
+
+    def test_exception_from_send(self):
+        def raiser():
+            while True:
+                v = yield
+                if v < 0:
+                    raise KeyError(v)
+
+        g = raiser()
+        next(g)
+        for i in range(WARM):
+            g.send(i)
+        with self.assertRaises(KeyError):
+            g.send(-1)
+        with self.assertRaises(StopIteration):
+            g.send(1)
+
+
+if __name__ == "__main__":
+    unittest.main()

From b4f447f8ed19726943c321f15e77b054fd7067e8 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 20:10:32 -0700
Subject: [PATCH 012/173] perf: resume compiled generators straight from native
 consumers

A compiled consumer's `SEND` or `FOR_ITER` that switches to a generator
or coroutine now runs the delegate's native code from the helper, as a
compiled caller runs a compiled callee (`run_switched_directly`, shared
with `run_callee_directly`): the delegate's yield or return switches
back through new native lowerings of a generator body's `YIELD_VALUE`
and `RETURN_VALUE` (`h_gen_yield`, `h_gen_return`, the core loop's arms
as helpers), and the consumer goes on in its own native code at the
`END_SEND`, past the `FOR_ITER`, or past the loop, without the two core
loop reloads per resume. Anything else (a fast step, a root activation
folding into a draining consumer, a yield passed up a chain, a pending
stop) leaves for the core loop as before.

A generator call moves its arguments into the new frame's locals with
one copy instead of through a draining iterator.

Instructions retired vs the branch point (suite, per call): coroutines
0.852, nqueens 0.882, generators_tree 0.942, comprehensions 0.994,
geomean 0.991. Recursive `await` probe: 2.72x -> 2.52x CPython.
---
 crates/weavepy-vm/src/frame_jit.rs            | 159 +++++++++++++-
 crates/weavepy-vm/src/lib.rs                  |  35 +++-
 .../regrtest/test_generator_direct_resume.py  | 195 ++++++++++++++++++
 3 files changed, 375 insertions(+), 14 deletions(-)
 create mode 100644 tests/regrtest/test_generator_direct_resume.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 3dae9a90..58340cd1 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -153,6 +153,7 @@ const S_SNAP: i32 = 56;
 const S_DEAD: i32 = 64;
 const S_OUT: i32 = 80;
 const S_INTERP: i32 = std::mem::offset_of!(State, interp) as i32;
+const S_SW: i32 = std::mem::offset_of!(State, sw) as i32;
 const S_FRAME: i32 = std::mem::offset_of!(State, frame) as i32;
 const _: () = {
     assert!(std::mem::offset_of!(State, locals) == S_LOCALS as usize);
@@ -1099,6 +1100,21 @@ unsafe extern "C" fn h_for_iter(st: *mut State, it: *mut Object, out: *mut Objec
                 *sw.last = st.last;
                 if !interp.core_gen_resume(sw, pc as usize, crate::InlineResume::ForIter) {
                     sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+                    return FOR_SWITCHED;
+                }
+                // A compiled body runs from here; its yield (the value on
+                // top, past the instruction) or its return (the iterator
+                // gone, past the loop) continues this code natively.
+                if run_switched_directly(st) {
+                    let frame = &*st.frame;
+                    let at = frame.pc as usize;
+                    let depth = frame.stack.len();
+                    if at == pc as usize + 1 && depth == len + 1 {
+                        return 1;
+                    }
+                    if depth == len - 1 && at == for_iter_exit(&frame.code, pc as usize) {
+                        return 2;
+                    }
                 }
             }
             return FOR_SWITCHED;
@@ -2477,6 +2493,34 @@ const DIRECT_CALL_DEPTH: u32 = 48;
 /// `st` is the caller's live state, its switch non-null, and the running
 /// activation the callee its `CALL` just pushed.
 unsafe fn run_callee_directly(st: &mut State, start: usize) -> bool {
+    // SAFETY: the caller's contract.
+    unsafe { run_switched_directly(st) && (*st.frame).stack.len() == start + 1 }
+}
+
+/// The pc a `FOR_ITER` at `pc` of `code` goes on at once its iterator is
+/// exhausted and gone: past the loop's `END_FOR` / `POP_ITER` pair.
+fn for_iter_exit(code: &CodeObject, pc: usize) -> usize {
+    let mut to = pc + 1 + code.instructions[pc].arg as usize;
+    let op = |pc: usize| code.instructions.get(pc).map(|i| i.op);
+    if op(to) == Some(OpCode::EndFor) {
+        to += 1;
+        if matches!(op(to), Some(OpCode::PopIter | OpCode::PopTop)) {
+            to += 1;
+        }
+    }
+    to
+}
+
+/// [`run_callee_directly`] for any activation just switched to (a call's
+/// callee, a resumed generator): `true` when its native code ran it until
+/// it switched back to this code's activation with nothing pending (the
+/// frame synced: the caller checks where it was left), `false` when the
+/// core loop goes on (with whatever the run left synced).
+///
+/// # Safety
+///
+/// As [`run_callee_directly`]'s.
+unsafe fn run_switched_directly(st: &mut State) -> bool {
     // SAFETY: the caller's contract.
     unsafe {
         let sw = &mut *st.sw;
@@ -2525,11 +2569,9 @@ unsafe fn run_callee_directly(st: &mut State, start: usize) -> bool {
         interp.direct_calls.set(nested);
         let sw = &mut *st.sw;
         if status == RELOAD {
-            // Returned into the caller, nothing pending: done. Otherwise
-            // whoever switched last synced what runs next.
-            return sw.pending.is_none()
-                && std::ptr::eq(sw.cur, st.frame)
-                && (*st.frame).stack.len() == start + 1;
+            // Back in the caller, nothing pending: done. Otherwise whoever
+            // switched last synced what runs next.
+            return sw.pending.is_none() && std::ptr::eq(sw.cur, st.frame);
         }
         // The callee stopped partway: its state, synced as the core loop
         // syncs it after a native run, for the core loop to go on with.
@@ -3572,8 +3614,11 @@ unsafe extern "C" fn h_await_iter(st: *mut State, at: *mut Object, kind: u64) ->
 
 /// `SEND` at `pc` of the running activation, its stack `len` deep: the
 /// generator or coroutine below the top resumed and switched to as the
-/// core loop's arm does (`RELOAD`); anything else (another delegate, or
-/// a generator's fast step, which never switches) declined untouched.
+/// core loop's arm does (`RELOAD`), or, a compiled delegate run from here
+/// that returned into this `SEND`, its result in the sent value's place
+/// for the code to go on at the `END_SEND` (`0`); anything else (another
+/// delegate, or a generator's fast step, which never switches) declined
+/// untouched.
 unsafe extern "C" fn h_send(st: *mut State, pc: u64, len: u64) -> u32 {
     // SAFETY: the code passes its live state and its stack depth (the
     // delegate and the sent value on top).
@@ -3597,6 +3642,70 @@ unsafe extern "C" fn h_send(st: *mut State, pc: u64, len: u64) -> u32 {
         let interp = &mut *st.interp.cast_mut();
         if !interp.core_gen_resume(sw, pc as usize, crate::InlineResume::Send) {
             sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+            return RELOAD;
+        }
+        if sw.pending.is_none() && !std::ptr::eq(sw.cur, st.frame) && run_switched_directly(st) {
+            let frame = &*st.frame;
+            let exit = pc as usize + 1 + frame.code.instructions[pc as usize].arg as usize;
+            if frame.pc as usize == exit && frame.stack.len() == len {
+                st.len = len;
+                return 0;
+            }
+        }
+        RELOAD
+    }
+}
+
+/// `YIELD_VALUE` at `pc` of the running activation, an inline generator
+/// resume (its stack `len` deep, the value on top), as the core loop's arm
+/// runs it: the generator parks and its consumer, the value delivered, is
+/// switched to (`RELOAD`); declined untouched for a generator's fast step,
+/// a root activation (a draining consumer's fold is the arm's), or a yield
+/// the switch doesn't take.
+unsafe extern "C" fn h_gen_yield(st: *mut State, pc: u64, len: u64) -> u32 {
+    // SAFETY: the code passes its live state and its stack depth.
+    unsafe {
+        let st = &mut *st;
+        if st.sw.is_null() || (*(*st.sw).inl).is_empty() {
+            return 1;
+        }
+        let sw = &mut *st.sw;
+        let frame = &mut *st.frame;
+        frame.stack.set_len(len as usize);
+        frame.pc = pc as u32;
+        *sw.last = st.last;
+        let interp = &mut *st.interp.cast_mut();
+        if !interp.core_gen_yield(sw, st.snap_gen) {
+            sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+        } else {
+            // A `yield from` chain passes the value straight up.
+            while sw.pending.is_none()
+                && interp.core_yield_chains(sw)
+                && interp.core_gen_yield(sw, st.snap_gen)
+            {}
+        }
+        RELOAD
+    }
+}
+
+/// `RETURN_VALUE` at `pc` of the running activation, an inline generator
+/// resume, as the core loop's arm runs it (see [`h_return`]); declined
+/// untouched for a generator's fast step or a root activation.
+unsafe extern "C" fn h_gen_return(st: *mut State, pc: u64, len: u64) -> u32 {
+    // SAFETY: the code passes its live state and its stack depth.
+    unsafe {
+        let st = &mut *st;
+        if st.sw.is_null() || (*(*st.sw).inl).is_empty() {
+            return 1;
+        }
+        let sw = &mut *st.sw;
+        let frame = &mut *st.frame;
+        frame.stack.set_len(len as usize);
+        frame.pc = pc as u32;
+        *sw.last = st.last;
+        let interp = &mut *st.interp.cast_mut();
+        if !interp.core_return(sw, st.snap_gen) {
+            sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
         }
         RELOAD
     }
@@ -5522,6 +5631,42 @@ impl<'a> Lower<'a> {
                     .expect("returns");
                 let out = self.exit_with(pc, &[], INTERP);
                 let switched = self.b.create_block();
+                let other = self.b.create_block();
+                let returned = self.b.create_block();
+                let reload = self.b.ins().icmp_imm(IntCC::Equal, r, i64::from(RELOAD));
+                self.b.ins().brif(reload, switched, &[], other, &[]);
+                self.b.switch_to_block(switched);
+                self.b.ins().return_(&[r]);
+                self.b.switch_to_block(other);
+                self.b.ins().brif(r, out, &[], returned, &[]);
+                // The delegate returned into this `SEND`: its result took the
+                // sent value's place.
+                self.b.switch_to_block(returned);
+                self.set_last(pc);
+                self.goto(next + ins.arg as usize);
+                return false;
+            }
+            // A generator body's yield and return switch to its consumer
+            // (`h_gen_yield`, `h_gen_return`); in a fast step (no switch)
+            // they're the step's own.
+            OpCode::YieldValue | OpCode::ReturnValue if self.is_gen() && self.depth > 0 => {
+                self.flush();
+                let sw = self.b.ins().load(self.ptr, FLAGS, self.st, S_SW);
+                let out = self.exit_with(pc, &[], INTERP);
+                let none = self.b.ins().icmp_imm(IntCC::Equal, sw, 0);
+                self.branch_out(none, out);
+                self.store_last();
+                let pcv = self.b.ins().iconst(types::I64, pc as i64);
+                let len = self.b.ins().iconst(types::I64, self.depth as i64);
+                let helper = if ins.op == OpCode::YieldValue {
+                    h_gen_yield as *const () as usize
+                } else {
+                    h_gen_return as *const () as usize
+                };
+                let r = self
+                    .call(helper, &[self.st, pcv, len], true)
+                    .expect("returns");
+                let switched = self.b.create_block();
                 let reload = self.b.ins().icmp_imm(IntCC::Equal, r, i64::from(RELOAD));
                 self.b.ins().brif(reload, switched, &[], out, &[]);
                 self.b.switch_to_block(switched);
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index e12116fc..ac7c4c02 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -56288,9 +56288,19 @@ impl Interpreter {
         // Committed: the arguments move into the generator's frame, and
         // the callee and its self slot leave the stack (one that dies
         // queues its finalizer, which runs before the next instruction).
-        let mut ops = frame.stack.drain(callee_at..);
-        let callee = ops.next().expect("checked above");
-        let slot = ops.next().expect("checked above");
+        // The operands are moved out whole: the stack forgets them first
+        // (nothing touches it until they're all moved).
+        let nargs = argc;
+        let base = frame.stack.as_mut_ptr();
+        // SAFETY: `callee_at + 2 + nargs` is the stack's length.
+        let (callee, slot, args) = unsafe {
+            frame.stack.set_len(callee_at);
+            (
+                base.add(callee_at).read(),
+                base.add(callee_at + 1).read(),
+                base.add(callee_at + 2),
+            )
+        };
         let (f, callee_receiver) = match &callee {
             Object::Function(f) => (f, None),
             Object::BoundMethod(bm) => match &bm.function {
@@ -56299,6 +56309,13 @@ impl Interpreter {
             },
             _ => unreachable!("checked above"),
         };
+        // Move the arguments onto the end of `v`.
+        // SAFETY: `args` holds `nargs` initialized values the stack forgot.
+        let take_args = |v: &mut Vec<Object>| unsafe {
+            v.reserve(nargs);
+            std::ptr::copy_nonoverlapping(args, v.as_mut_ptr().add(v.len()), nargs);
+            v.set_len(v.len() + nargs);
+        };
         let gen = if let Some(cells) = cells {
             // The arguments move straight into the locals (no staging).
             let nlocals = code.varnames.len();
@@ -56307,8 +56324,10 @@ impl Interpreter {
                 // SAFETY: sole owner (see `pooled_locals_empty`).
                 let v = unsafe { &mut *locals.as_ptr() };
                 v.extend(callee_receiver);
-                v.extend((!matches!(slot, Object::Unbound)).then_some(slot));
-                v.extend(ops);
+                if !matches!(slot, Object::Unbound) {
+                    v.push(slot);
+                }
+                take_args(v);
                 let missing = (code.arg_count as usize).saturating_sub(v.len());
                 if missing > 0 {
                     v.extend(f.defaults[f.defaults.len() - missing..].iter().cloned());
@@ -56322,8 +56341,10 @@ impl Interpreter {
         } else {
             let mut positional = self.pooled_scratch();
             positional.extend(callee_receiver);
-            positional.extend((!matches!(slot, Object::Unbound)).then_some(slot));
-            positional.extend(ops);
+            if !matches!(slot, Object::Unbound) {
+                positional.push(slot);
+            }
+            take_args(&mut positional);
             self.start_generator_at(f, &code, positional, start)
         };
         self.release(callee);
diff --git a/tests/regrtest/test_generator_direct_resume.py b/tests/regrtest/test_generator_direct_resume.py
new file mode 100644
index 00000000..8c7c9b56
--- /dev/null
+++ b/tests/regrtest/test_generator_direct_resume.py
@@ -0,0 +1,195 @@
+"""Compiled generator bodies resumed straight from compiled consumers: a
+`FOR_ITER` or `SEND` in native code runs the generator's native code
+itself, and its yield or return comes back to the consumer's native code
+(see `run_switched_directly` in the VM's frame JIT).
+
+Each case warms a consumer and a generator that fast steps don't take
+(a call or a handler in the body) until both compile, then checks values,
+exhaustion, raises, return values through `yield from` and `await`,
+frames seen from inside the generator, finalizers, and recursion limits.
+"""
+
+import sys
+import unittest
+
+WARM = 3000
+
+
+def double(x):
+    return x * 2
+
+
+def calls(n):
+    for i in range(n):
+        yield double(i)
+
+
+def guarded(xs):
+    for x in xs:
+        try:
+            yield 10 // x
+        except ZeroDivisionError:
+            yield None
+
+
+def consume(gen):
+    total = 0
+    for x in gen:
+        total += x
+    return total
+
+
+def inner(n):
+    for i in range(n):
+        yield double(i)
+    return n
+
+
+def outer(n):
+    got = yield from inner(n)
+    yield got
+
+
+async def leaf(x):
+    return double(x)
+
+
+async def node(x):
+    return await leaf(x) + await leaf(x + 1)
+
+
+def drive(coro):
+    try:
+        coro.send(None)
+    except StopIteration as e:
+        return e.value
+
+
+class DirectResumeTests(unittest.TestCase):
+    def test_values_and_exhaustion(self):
+        for _ in range(WARM):
+            consume(calls(5))
+        self.assertEqual(consume(calls(10)), sum(2 * i for i in range(10)))
+        self.assertEqual(consume(calls(0)), 0)
+        self.assertEqual(list(calls(3)), [0, 2, 4])
+
+    def test_raise_inside(self):
+        for _ in range(WARM):
+            list(guarded([1, 2, 0, 5]))
+        self.assertEqual(list(guarded([1, 0, 2])), [10, None, 5])
+
+        def bad(n):
+            for i in range(n):
+                yield double(i)
+            raise KeyError("end")
+
+        for _ in range(WARM):
+            try:
+                consume(bad(3))
+            except KeyError:
+                pass
+        with self.assertRaises(KeyError):
+            consume(bad(3))
+        with self.assertRaises(RuntimeError):
+            # PEP 479: a StopIteration escaping the body.
+            def stopper():
+                yield double(1)
+                raise StopIteration
+
+            consume(stopper())
+
+    def test_yield_from_return_value(self):
+        for _ in range(WARM):
+            list(outer(3))
+        self.assertEqual(list(outer(3)), [0, 2, 4, 3])
+
+    def test_await_chain(self):
+        for i in range(WARM):
+            drive(node(i))
+        self.assertEqual(drive(node(5)), 22)
+
+    def test_frames_from_inside(self):
+        def where():
+            for _ in range(2):
+                f = sys._getframe()
+                yield (f.f_code.co_name, f.f_back.f_code.co_name if f.f_back else None)
+
+        def run():
+            out = []
+            for item in where():
+                out.append(item)
+            return out
+
+        for _ in range(WARM):
+            run()
+        self.assertEqual(run(), [("where", "run"), ("where", "run")])
+
+    def test_finalizer_on_exhaustion(self):
+        log = []
+
+        class Tracked:
+            def __del__(self):
+                log.append("del")
+
+        def holder():
+            t = Tracked()
+            yield double(1)
+            del t
+            log.append("after")
+            yield double(2)
+
+        for _ in range(WARM):
+            consume(holder())
+        log.clear()
+        for x in holder():
+            log.append(x)
+        self.assertEqual(log, [2, "del", "after", 4])
+
+    def test_recursion_limit(self):
+        def nest(n):
+            if n:
+                yield from nest(n - 1)
+            yield double(n)
+
+        for _ in range(200):
+            consume(nest(20))
+        limit = sys.getrecursionlimit()
+        try:
+            sys.setrecursionlimit(200)
+            with self.assertRaises(RecursionError):
+                consume(nest(400))
+        finally:
+            sys.setrecursionlimit(limit)
+        self.assertEqual(consume(nest(50)), sum(2 * i for i in range(51)))
+
+    def test_close_and_throw_midway(self):
+        def counting():
+            try:
+                for i in range(10):
+                    yield double(i)
+            finally:
+                log.append("closed")
+
+        log = []
+        for _ in range(WARM):
+            g = counting()
+            for x in g:
+                if x > 4:
+                    break
+            g.close()
+        log.clear()
+        g = counting()
+        for x in g:
+            if x > 4:
+                break
+        g.close()
+        self.assertEqual(log, ["closed"])
+        g = counting()
+        next(g)
+        with self.assertRaises(ValueError):
+            g.throw(ValueError)
+        self.assertEqual(log, ["closed", "closed"])
+
+
+if __name__ == "__main__":
+    unittest.main()

From 4e617f9f8037f25b1de6545b39f45ccc0003d4c3 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 20:35:03 -0700
Subject: [PATCH 013/173] perf: recycle generator frame boxes; warm only
 delegating bodies

A finished generator's frame box goes back to a small per-interpreter
pool (its frame dropped in place) and the next generator's frame is
written into it, so a generator's life no longer costs a box malloc and
free, and the new frame is boxed once instead of being moved through
`wrap_started_generator` by value.

Warming a generator-family body at its creation (so its `SEND`s run
natively) is now limited to coroutines and bodies that await or
delegate (a `SEND`, marked in the cached `RETURN_GENERATOR` verdict): a
plain loop-free generator such as a `@contextmanager` body ran slower
compiled than interpreted (context_managers +0.9%), and plain
generators still warm by their loops and fast steps.

Instructions retired vs the previous commit (suite, per call):
coroutines 0.982, async_tree 0.986, context_managers 0.991,
generators_tree 0.990, nqueens 0.994, geomean 0.997.
---
 crates/weavepy-vm/src/gen_fast.rs |  2 +-
 crates/weavepy-vm/src/lib.rs      | 75 +++++++++++++++++++++++++------
 2 files changed, 63 insertions(+), 14 deletions(-)

diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index 2d8c0aff..c89ea35a 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -1079,7 +1079,7 @@ impl Interpreter {
                 let frame: &mut Frame = &mut boxed;
                 self.recycle_frame_allocs(frame);
                 Self::release_finished_gen(g);
-                drop(boxed);
+                self.recycle_gen_box(boxed);
                 return GenNext::Exhausted(v);
             }
             GenStep::Bail if frame.pc == start => {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ac7c4c02..b4904633 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -922,6 +922,10 @@ pub struct Interpreter {
     /// invalidate on every growth.
     #[allow(clippy::vec_box)]
     inline_pool: Vec<Box<InlineAct>>,
+    /// Finished generators' frame boxes, their frames dropped, for the
+    /// next generators' frames (see [`Self::box_gen_frame`]).
+    #[allow(clippy::vec_box)]
+    gen_box_pool: ThreadCell<Vec<Box<std::mem::MaybeUninit<Frame>>>>,
     /// RFC 0068 (WS9) — recycled tuple allocations, CPython's tuple
     /// freelist analogue (one LIFO bucket per length 1..=16, like
     /// CPython's per-size freelists). A refcount-dead, untracked,
@@ -1224,6 +1228,7 @@ impl Default for Interpreter {
             internal_import_depth: 0,
             frame_locals_pool: ThreadCell::new(Vec::new()),
             inline_pool: Vec::new(),
+            gen_box_pool: ThreadCell::new(Vec::new()),
             tuple_pool: ThreadCell::new(std::array::from_fn(|_| Vec::new())),
             frame_shell_pool: ThreadCell::new(Vec::new()),
             frame_stack_pool: ThreadCell::new(Vec::new()),
@@ -1352,6 +1357,7 @@ impl Interpreter {
             internal_import_depth: 0,
             frame_locals_pool: ThreadCell::new(Vec::new()),
             inline_pool: Vec::new(),
+            gen_box_pool: ThreadCell::new(Vec::new()),
             tuple_pool: ThreadCell::new(std::array::from_fn(|_| Vec::new())),
             frame_shell_pool: ThreadCell::new(Vec::new()),
             frame_stack_pool: ThreadCell::new(Vec::new()),
@@ -3977,6 +3983,36 @@ impl Interpreter {
         }
     }
 
+    /// `frame` boxed for a new generator: in a finished generator's box
+    /// (see [`Self::recycle_gen_box`]) when the pool has one.
+    #[inline]
+    fn box_gen_frame(&self, frame: Frame) -> Box<Frame> {
+        let spare = self.gen_box_pool.borrow_mut().pop();
+        match spare {
+            Some(b) => Box::write(b, frame),
+            None => Box::new(frame),
+        }
+    }
+
+    /// A finished generator's frame box: the frame is dropped here (what
+    /// dropping the box would do) and the allocation kept for the next
+    /// generator's (see [`Self::box_gen_frame`]).
+    fn recycle_gen_box(&self, boxed: Box<Frame>) {
+        const POOL_CAP: usize = 32;
+        let raw = Box::into_raw(boxed);
+        // SAFETY: the frame is dropped in place once, and its allocation
+        // goes on as uninitialized storage (freed as such past the cap).
+        // The frame's drop glue runs before the pool's borrow.
+        let spare = unsafe {
+            std::ptr::drop_in_place(raw);
+            Box::from_raw(raw.cast::<std::mem::MaybeUninit<Frame>>())
+        };
+        let mut pool = self.gen_box_pool.borrow_mut();
+        if pool.len() < POOL_CAP {
+            pool.push(spare);
+        }
+    }
+
     fn make_frame(
         &self,
         code: Rc<CodeObject>,
@@ -20625,7 +20661,7 @@ impl Interpreter {
         *gen.state.borrow_mut() = GeneratorState::Finished;
         self.recycle_frame_allocs(&mut boxed);
         Self::release_finished_gen(&gen);
-        drop(boxed);
+        self.recycle_gen_box(boxed);
         drop(gen);
         // A collapsed chain's innermost level takes the return value: it
         // resumes in its `SEND`, the rest of the chain still collapsed.
@@ -43166,6 +43202,7 @@ impl Interpreter {
                 let frame: &mut Frame = &mut boxed;
                 self.recycle_frame_allocs(frame);
                 Self::release_finished_gen(gen);
+                self.recycle_gen_box(boxed);
                 Self::generator_returned(v, raw_return)
             }
             Ok(FrameOutcome::StartGenerator) => {
@@ -55980,7 +56017,7 @@ impl Interpreter {
             };
             match bootstrap {
                 FrameOutcome::StartGenerator => {
-                    let obj = Self::wrap_started_generator(f, &code, frame);
+                    let obj = Self::wrap_started_generator(f, &code, self.box_gen_frame(frame));
                     if let Some(origin) = cr_origin {
                         if let Object::Coroutine(gen) = &obj {
                             gen.set_origin(origin);
@@ -56016,7 +56053,7 @@ impl Interpreter {
 
     /// Wrap `frame`, `f`'s generator-family body just past
     /// `RETURN_GENERATOR`, in its generator object (tracked).
-    fn wrap_started_generator(f: &PyFunction, code: &CodeObject, frame: Frame) -> Object {
+    fn wrap_started_generator(f: &PyFunction, code: &CodeObject, frame: Box<Frame>) -> Object {
         let kind = if code.is_coroutine {
             crate::object::CoroutineKind::Coroutine
         } else if code.is_async_generator {
@@ -56029,11 +56066,16 @@ impl Interpreter {
         // `gi_name`/`gi_qualname` at call time, sharing the objects.
         // A generator-family body's activations never start at pc 0, where
         // the core loop counts a call's heat (see `frame_jit::Slot::warm`):
-        // its creation counts instead.
+        // the creation of one that awaits or delegates counts instead (its
+        // `SEND`s run natively; a plain generator's body still warms by
+        // its loops and fast steps, which pays off more reliably).
         #[cfg(feature = "jit")]
         if let Some(ext) = code_vm_ext(code) {
-            ext.frame_jit
-                .warm(code, ext, code.varnames.len(), frame_jit::Heat::Call);
+            let start = ext.gen_start.load(std::sync::atomic::Ordering::Relaxed);
+            if code.is_coroutine || (start != u32::MAX && start & GEN_DELEGATES != 0) {
+                ext.frame_jit
+                    .warm(code, ext, code.varnames.len(), frame_jit::Heat::Call);
+            }
         }
         let gen_code = Object::Code(frame.code.clone());
         let mut frame = frame;
@@ -56043,7 +56085,7 @@ impl Interpreter {
             frame.gen_owner = Some(owner.clone());
             if f.names_seeded() {
                 // The code's names, which the generator reads by default.
-                PyGenerator::new(kind, gen_code, Box::new(frame))
+                PyGenerator::new(kind, gen_code, frame)
             } else {
                 let (name, qualname) = f.name_objects();
                 PyGenerator::with_names(
@@ -56051,7 +56093,7 @@ impl Interpreter {
                     qualname.unwrap_or_else(|| Object::from_str(code.qualname.clone())),
                     kind,
                     gen_code,
-                    Box::new(frame),
+                    frame,
                 )
             }
         });
@@ -56118,7 +56160,7 @@ impl Interpreter {
         let start = match ext.gen_start.load(std::sync::atomic::Ordering::Relaxed) {
             0 => Self::generator_start_scan(code, ext)?,
             u32::MAX => return None,
-            at => at as usize - 1,
+            at => (at & !GEN_DELEGATES) as usize - 1,
         };
         let nargs = code.arg_count as usize;
         if given > nargs
@@ -56152,8 +56194,11 @@ impl Interpreter {
             .iter()
             .position(|i| !matches!(i.op, OpCode::MakeCell | OpCode::CopyFreeVars))
             .filter(|&start| shaped && code.instructions[start].op == OpCode::ReturnGenerator);
+        let delegates = code.instructions.iter().any(|i| i.op == OpCode::Send);
         ext.gen_start.store(
-            start.map_or(u32::MAX, |at| at as u32 + 1),
+            start.map_or(u32::MAX, |at| {
+                (at as u32 + 1) | if delegates { GEN_DELEGATES } else { 0 }
+            }),
             std::sync::atomic::Ordering::Relaxed,
         );
         start
@@ -56193,7 +56238,7 @@ impl Interpreter {
                 frame
             }
         };
-        Self::wrap_started_generator(f, code, frame)
+        Self::wrap_started_generator(f, code, self.box_gen_frame(frame))
     }
 
     /// The frame of a generator-family call of `f` (its code `code`, which
@@ -56337,7 +56382,7 @@ impl Interpreter {
             let mut gen_frame = Self::gen_frame_lean(f, code, locals, cells, start);
             gen_frame.stack = self.pooled_stack();
             let code = Rc::clone(&gen_frame.code);
-            Self::wrap_started_generator(f, &code, gen_frame)
+            Self::wrap_started_generator(f, &code, self.box_gen_frame(gen_frame))
         } else {
             let mut positional = self.pooled_scratch();
             positional.extend(callee_receiver);
@@ -70853,7 +70898,8 @@ struct CodeConstObjects {
     /// The `RETURN_GENERATOR` a generator-family call of the code starts
     /// past (see `Interpreter::generator_start`): `0` not yet scanned,
     /// `u32::MAX` none (the prologue holds something else), else its pc
-    /// plus one.
+    /// plus one, with [`GEN_DELEGATES`] set for a body that awaits or
+    /// delegates (a `SEND`).
     gen_start: std::sync::atomic::AtomicU32,
     /// Split-layout attribute shortcuts per `LOAD_ATTR` site (see
     /// [`FieldSlot`]); allocated on the first recorded one.
@@ -70900,6 +70946,9 @@ struct CodeConstObjects {
     dispatch_len: usize,
 }
 
+/// [`CodeConstObjects::gen_start`]'s mark of a body with a `SEND`.
+const GEN_DELEGATES: u32 = 1 << 31;
+
 /// The bound the core loop checks a code object's entry pc against, so it
 /// dispatches each instruction without a bounds check: the instruction
 /// count when no path from an instruction can leave the code, else `0`

From 3c3fbc71bdc8cce9abc8a3f6aa47f707fd36d5b4 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 20:10:58 -0700
Subject: [PATCH 014/173] perf: run container ops and builtin methods in place
 in the frame JIT

The frame JIT handed most container instructions and builtin method calls
to generic helpers that copied every operand onto the stack, released them
after, and dispatched on the operation at run time. This change lowers the
common shapes to lean helpers that read local and constant operands in
place, and gives builtin methods per-method kernels:

- Method kernels: `x.m(args)` on a local exact `list`, `dict`, `set` or
  `str` whose site settled on a leaf method checks the receiver's tag in
  line and calls a kernel on the operands where they lie (no method
  object, receiver copy or argument copies), continuing after the `CALL`.
  Kernels cover `append`, `pop`, `extend` (a new leaf kind), `dict.get`,
  `set.add`/`discard`/`remove`, `startswith`/`endswith`/`find` (with a
  start position), `isdigit`/`isalpha`/`isspace` on ASCII text, and the
  pure `str` bodies (`lower`, `strip`, `split`, `join`, `replace`, ...).
  Calls of the same methods on stack receivers use them through a small
  leaf-site helper, which also runs `isinstance`, `len` and the registered
  fast halves before the general call helper.
- `BINARY_SUBSCR`, `STORE_SUBSCR`, `UNPACK_SEQUENCE` and `CONTAINS_OP`
  read local and constant operands in place, settle the exact-container
  shapes in one small helper, and fall back to the container helper
  inside it (in lean bodies too). Unpacking a dead tuple recycles it into
  the tuple free list, and `enumerate` steps draw from it.
- Copies and releases of strings and tuples are in-line count changes,
  as for instances and lists (their counts sit two words before the
  payload); builtins and sets join the counted kinds; `POP_TOP` releases
  a shared value by a plain decrement.
- `FOR_ITER` takes a live range or a list's, tuple's or ASCII string's
  next item in a small helper before the general one.
- A method load from a module keeps the entry it read while the module's
  dict stamp holds (`math.floor`).
- A loop whose calls are settled leaf builtins counts them as run in
  place when deciding to compile.
- Hashing: `int` and `str` keys hash without the general dispatch, a
  tuple of ints and strings without an error scope, and `hash()` is a
  fast leaf for such values.
- `str.strip`/`lstrip`/`rstrip` scan bytes for whitespace or an ASCII
  set (one allocation, the receiver itself when nothing goes), and a
  one-byte needle in a short string is a plain scan.

Instructions per op (loop overhead subtracted, operands as locals),
before -> after (CPython 3.14): set.add+discard 2269 -> 711 (617);
list.append+pop 917 -> 200 (530); x = []; x.extend(t) 2754 -> 1035
(681); hash((1, 2, i)) 2384 -> 1024 (776); s.isdigit() 616 -> 150 (239);
s.startswith('12') 630 -> 145 (338); d.get('a') 651 -> 300 (280);
d[key] on an 81-key dict 625 -> 400 (233); '5' in s 404 -> 215 (200);
a, b, c = t 397 -> 215 (124); for c in 'abcdef' 2817 -> 1590 (1099);
isinstance(l, list) 990 -> 650 (248); math.floor(2.5) 846 -> 553 (207).

Suite (instructions per call vs the branch's baseline): geomean 0.977;
unpack_sequence 0.89, graph_search 0.90, sudoku 0.90, text_processing
0.91, mini_interpreter 0.92, nqueens 0.93, chaos 0.94, tomllib_loads
0.95, go 0.97; regex_mix 1.016 (its bench loop now compiles), and
float_points and collections_mix 1.016 (unexplained: neither runs the
changed paths much).
---
 crates/weavepy-vm/src/builtins.rs           |   63 +
 crates/weavepy-vm/src/frame_jit.rs          | 1643 ++++++++++++++++++-
 crates/weavepy-vm/src/lib.rs                |   57 +
 crates/weavepy-vm/src/object.rs             |   34 +-
 crates/weavepy-vm/src/seqiter.rs            |   21 +
 crates/weavepy-vm/src/shared_value.rs       |    9 +
 tests/regrtest/test_frame_jit_containers.py |  228 +++
 7 files changed, 1996 insertions(+), 59 deletions(-)
 create mode 100644 tests/regrtest/test_frame_jit_containers.py

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 94637a4b..4e8e733b 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -11008,7 +11008,59 @@ fn str_casefold(args: &[Object]) -> Result<Object, RuntimeError> {
     Ok(str_result(args, out))
 }
 
+/// `strip`/`lstrip`/`rstrip` of an exact `str` by whitespace or by a set
+/// of ASCII characters, scanning bytes: the receiver itself when nothing
+/// goes, else one allocation. `None` for the general path (a non-ASCII
+/// set, or a non-ASCII character at an edge whitespace stripping reaches).
+fn strip_fast(args: &[Object], left: bool, right: bool) -> Option<Object> {
+    let (Some(Object::Str(s)), true) = (args.first(), args.len() <= 2) else {
+        return None;
+    };
+    let spaces = matches!(args.get(1), None | Some(Object::None));
+    let mut set = [false; 128];
+    match args.get(1) {
+        None | Some(Object::None) => set = *ascii_space_table(),
+        Some(Object::Str(chars)) if chars.is_ascii() => {
+            for &c in chars.as_bytes() {
+                set[usize::from(c)] = true;
+            }
+        }
+        _ => return None,
+    }
+    let b = s.as_bytes();
+    let strips = |c: u8| c < 0x80 && set[usize::from(c)];
+    let (mut lo, mut hi) = (0, b.len());
+    if left {
+        while lo < hi && strips(b[lo]) {
+            lo += 1;
+        }
+    }
+    if right {
+        while hi > lo && strips(b[hi - 1]) {
+            hi -= 1;
+        }
+    }
+    // (A non-ASCII character may be Unicode whitespace.)
+    if spaces && ((left && lo < hi && b[lo] >= 0x80) || (right && hi > lo && b[hi - 1] >= 0x80)) {
+        return None;
+    }
+    if lo == 0 && hi == b.len() {
+        return Some(args[0].clone());
+    }
+    // (The shortest results are the shared ones `SharedStr::from` hands
+    // out, as `str_result` would.)
+    let ascii = hi - lo > 2 && SharedStr::char_count(s) == b.len();
+    Some(Object::Str(if ascii {
+        SharedStr::from_ascii(&s[lo..hi])
+    } else {
+        SharedStr::from(&s[lo..hi])
+    }))
+}
+
 fn str_strip(args: &[Object]) -> Result<Object, RuntimeError> {
+    if let Some(out) = strip_fast(args, true, true) {
+        return Ok(out);
+    }
     str_arity("strip", args, 0, 1)?;
     let s = str_self(args)?;
     let out = match args.get(1) {
@@ -11676,6 +11728,11 @@ pub(crate) fn substr_find(hay: &str, needle: &str) -> Option<usize> {
     if n.is_empty() {
         return Some(0);
     }
+    // One ASCII byte in a short string (`d in "123456789"`): a plain scan
+    // costs less than `memchr`'s set-up.
+    if let ([byte], true) = (n, h.len() <= 16) {
+        return h.iter().position(|c| c == byte);
+    }
     if n.len() > 16 || h.len() < n.len() {
         return hay.find(needle);
     }
@@ -11929,6 +11986,9 @@ fn str_swapcase(args: &[Object]) -> Result<Object, RuntimeError> {
 }
 
 fn str_lstrip(args: &[Object]) -> Result<Object, RuntimeError> {
+    if let Some(out) = strip_fast(args, true, false) {
+        return Ok(out);
+    }
     str_arity("lstrip", args, 0, 1)?;
     let s = str_self(args)?;
     let out = match args.get(1) {
@@ -11949,6 +12009,9 @@ fn str_lstrip(args: &[Object]) -> Result<Object, RuntimeError> {
 }
 
 fn str_rstrip(args: &[Object]) -> Result<Object, RuntimeError> {
+    if let Some(out) = strip_fast(args, false, true) {
+        return Ok(out);
+    }
     str_arity("rstrip", args, 0, 1)?;
     let s = str_self(args)?;
     let out = match args.get(1) {
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 58340cd1..f2727c50 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -200,6 +200,10 @@ pub(crate) struct Native {
     /// The `CALL` sites' direct-call caches.
     #[allow(clippy::vec_box)]
     _direct: Vec<Box<DirectSite>>,
+    /// The builtins the method kernels' calls address.
+    _held: Vec<Rc<crate::object::BuiltinFn>>,
+    #[allow(clippy::vec_box)]
+    _mods: Vec<Box<ModCache>>,
 }
 
 // SAFETY: the code only touches the state it is handed; it is only
@@ -408,6 +412,13 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
     // A method call on a local runs through a helper without leaving, when
     // its callee runs in place.
     let mut helped = vec![false; ins.len()];
+    // So does a call of a leaf builtin (`isinstance(x, C)`, `len(xs)`,
+    // `math.floor(x)`) the site has settled.
+    for (pc, h) in helped.iter_mut().enumerate() {
+        if ins[pc].op == OpCode::Call && crate::call_site_in_place(ext, pc) {
+            *h = true;
+        }
+    }
     for pc in 0..ins.len().saturating_sub(1) {
         if ins[pc].op == OpCode::LoadFast
             && ins[pc + 1].op == OpCode::LoadMethodAttr
@@ -791,10 +802,18 @@ struct Tags {
     /// payload word: `(length, first item)` byte offsets.
     tuple: u8,
     tuple_layout: Option<(i32, i32)>,
+    /// The `dict`, `set` and `str` tags (the method kernels' receivers).
+    dict: u8,
+    set: u8,
+    str: u8,
     /// The heap variants whose payload word is an `Rc`'s allocation, its
     /// strong count the allocation's first word (a clone is an increment
     /// there): instances, lists, dicts and classes, as far as measured.
     rc_counted: i64,
+    /// The heap variants whose payload word addresses a shared payload
+    /// with its strong count two words before it (see `rc::strong_word`):
+    /// strings and tuples, as far as measured.
+    rc_counted_hdr: i64,
     /// The heap variants whose identity is their payload pointer (and so
     /// `is` compares payload words): instances, strings, lists, dicts,
     /// tuples, classes.
@@ -874,11 +893,30 @@ fn tags() -> Option<Tags> {
                 &Object::new_dict(),
                 &Object::Type(crate::builtin_types::builtin_types().object_.clone()),
                 &func,
+                &Object::new_set(),
+                &Object::Builtin(Rc::new(crate::object::BuiltinFn::new("sample", |_| {
+                    Ok(Object::None)
+                }))),
             ]
             .into_iter()
             .filter(|o| tag(o) < 64 && counted_at_payload(o, word(o)))
             .fold(0, |m, o| m | (1i64 << tag(o)))
         },
+        dict: tag(&Object::new_dict()),
+        set: tag(&Object::new_set()),
+        str: tag(&Object::from_str("")),
+        rc_counted_hdr: if cfg!(debug_assertions) {
+            0
+        } else {
+            // (A fresh string: the small ones are shared statics.)
+            [
+                &Object::from_str("not a small string"),
+                &Object::new_tuple_array([Object::None, Object::None]),
+            ]
+            .into_iter()
+            .filter(|o| tag(o) < 64 && word(o) >= 16 && counted_at_payload(o, word(o) - 16))
+            .fold(0, |m, o| m | (1i64 << tag(o)))
+        },
         by_pointer: samples
             .iter()
             .fold(0, |m, &t| if t < 64 { m | (1i64 << t) } else { m }),
@@ -1037,8 +1075,72 @@ unsafe extern "C" fn h_store(st: *mut State, slot: *mut Object, i: u64) -> u32 {
 /// `FOR_ITER` over a range, list or tuple iterator at `it` (the next value
 /// goes to `out`, the slot above): `0` an `int` in `st.out`, `1` another
 /// value written to `out`, `2` the exhausted iterator retired (it left the
-/// stack), anything else declined.
+/// stack), anything else declined. The commonest steps (a live range, a
+/// list's or tuple's next item) run here; everything else (exhaustion, a
+/// generator, the other iterators) in [`h_for_iter_rest`].
 unsafe extern "C" fn h_for_iter(st: *mut State, it: *mut Object, out: *mut Object, pc: u64) -> u32 {
+    // SAFETY: as `h_for_iter_rest`'s; nothing here runs code while the
+    // iterator is borrowed.
+    unsafe {
+        if let Object::Iter(rc) = &*it {
+            if let Some(iter) = rc.peek_mut() {
+                let v = match iter {
+                    PyIterator::Range {
+                        current,
+                        stop,
+                        step,
+                    } if *step > 0 && *current < *stop => {
+                        (*st).out = *current as u64;
+                        *current = current.wrapping_add(*step);
+                        return 0;
+                    }
+                    PyIterator::List { items, index, .. } => {
+                        match items.peek().and_then(|xs| xs.get(*index)) {
+                            Some(v) => {
+                                *index += 1;
+                                v
+                            }
+                            None => return h_for_iter_rest(st, it, out, pc),
+                        }
+                    }
+                    PyIterator::Tuple { items, index } => match items.get(*index) {
+                        Some(v) => {
+                            *index += 1;
+                            v
+                        }
+                        None => return h_for_iter_rest(st, it, out, pc),
+                    },
+                    // A string's next ASCII character (a shared one).
+                    PyIterator::Str { s, index } => match s.as_bytes().get(*index) {
+                        Some(&c) if c < 0x80 => {
+                            *index += 1;
+                            out.write(Object::Str(crate::shared_value::SharedStr::ascii_char(c)));
+                            return 1;
+                        }
+                        _ => return h_for_iter_rest(st, it, out, pc),
+                    },
+                    _ => return h_for_iter_rest(st, it, out, pc),
+                };
+                if let Object::Int(i) = *v {
+                    (*st).out = i as u64;
+                    return 0;
+                }
+                out.write(crate::clone_hot(v));
+                return 1;
+            }
+        }
+        h_for_iter_rest(st, it, out, pc)
+    }
+}
+
+/// [`h_for_iter`]'s other shapes.
+#[inline(never)]
+unsafe extern "C" fn h_for_iter_rest(
+    st: *mut State,
+    it: *mut Object,
+    out: *mut Object,
+    pc: u64,
+) -> u32 {
     // SAFETY: the code passes its live state, an initialized slot with a
     // free one above it, and its `FOR_ITER`'s pc.
     let (st, top) = unsafe { (&mut *st, &*it) };
@@ -1288,7 +1390,10 @@ unsafe extern "C" fn h_for_iter(st: *mut State, it: *mut Object, out: *mut Objec
             };
             let i = *count;
             *count += 1;
-            Object::new_tuple_array([Object::Int(i), x])
+            // (From the tuple free list.)
+            // SAFETY: the running thread's interpreter, dormant while the
+            // helper runs.
+            unsafe { (*st.interp).alloc_tuple_array([Object::Int(i), x]) }
         }
         // A set's next element while the set keeps its size (a change,
         // and exhaustion, are the core loop's to report).
@@ -1357,6 +1462,588 @@ unsafe extern "C" fn h_list_append(list: *const Object, value: *const Object) ->
     }
 }
 
+/// A method kernel: `recv.m(a, b)` for an exact `list`, `dict`, `set` or
+/// `str` receiver (its tag checked by the caller), with the arguments the
+/// site passes (null past them) read in place, and the method's builtin
+/// `f` (for the kernels that run its body): `0` the result written to the
+/// free slot `out`, anything else declined untouched (the call then runs
+/// as before). A kernel runs no Python code, raises nothing, and releases
+/// nothing that could run a finalizer.
+type Kernel = unsafe extern "C" fn(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    f: *const crate::object::BuiltinFn,
+) -> u32;
+
+/// The kernel for the `tag` receiver's method of leaf kind `kind` called
+/// with `nargs` arguments (see [`Lower::method_kernel`]).
+fn kernel_for(tag: u64, kind: crate::LeafKind, nargs: usize) -> Option<Kernel> {
+    use crate::LeafKind as K;
+    Some(match (tag, kind, nargs) {
+        (1, K::ListAppend, 1) => k_list_append,
+        (1, K::ListPop, 0) => k_list_pop,
+        (1, K::ListExtend, 1) => k_list_extend,
+        (2, K::DictGet, 1 | 2) => k_dict_get,
+        (3, K::SetAdd, 1) => k_set_add,
+        (3, K::SetDiscard, 1) => k_set_discard,
+        (3, K::SetRemove, 1) => k_set_remove,
+        (4, K::StrStartswith, 1 | 2) => k_str_startswith,
+        (4, K::StrEndswith, 1 | 2) => k_str_endswith,
+        (4, K::StrIsdigit, 0) => k_str_isdigit,
+        (4, K::StrIsalpha, 0) => k_str_isalpha,
+        (4, K::StrIsspace, 0) => k_str_isspace,
+        (4, K::StrFind, 1 | 2) => k_str_find,
+        (4, K::StrLower | K::StrUpper, 0) => k_str_body,
+        (4, K::StrStrip | K::StrLstrip | K::StrRstrip, 0 | 1) => k_str_strip,
+        (4, K::StrSplit, 0 | 1) => k_str_split,
+        (4, K::StrJoin, 1) => k_str_join,
+        (4, K::StrReplace, 2) => k_str_replace,
+        (4, K::Opaque, _) => k_str_body,
+        _ => return None,
+    })
+}
+
+/// The call of method body `b` of leaf kind `kind` on `args` (the
+/// receiver first) through its [`Kernel`], for a `list`, `dict`, `set` or
+/// `str` receiver that has one: its result, or `None` (nothing done).
+/// (Out of line: the general call helper stays as it was.)
+#[inline(never)]
+fn kernel_call(
+    kind: crate::LeafKind,
+    args: &[Object],
+    b: &Rc<crate::object::BuiltinFn>,
+) -> Option<Object> {
+    let tag = match args.first()? {
+        Object::List(_) => 1,
+        Object::Dict(_) => 2,
+        Object::Set(_) => 3,
+        Object::Str(_) => 4,
+        _ => return None,
+    };
+    if args.len() > 3 {
+        return None;
+    }
+    let kernel = kernel_for(tag, kind, args.len() - 1)?;
+    let arg = |k: usize| args.get(k).map_or(std::ptr::null(), std::ptr::from_ref);
+    let mut out = std::mem::MaybeUninit::<Object>::uninit();
+    // SAFETY: the operands are initialized values; `out` is free.
+    unsafe {
+        (kernel(out.as_mut_ptr(), &args[0], arg(1), arg(2), Rc::as_ptr(b)) == 0)
+            .then(|| out.assume_init())
+    }
+}
+
+/// The method's own body on bitwise views of the operands (as the core
+/// loop's leaf call runs it): `0` its result written to `out`, anything
+/// else (an error, which the full call raises) declined.
+///
+/// # Safety
+///
+/// As [`Kernel`]'s, for a builtin whose body runs no Python code on
+/// these operands.
+unsafe fn run_body(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    f: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: the caller's contract; the views are never dropped.
+    unsafe {
+        let views = std::mem::ManuallyDrop::new([
+            recv.read(),
+            if a.is_null() {
+                Object::Unbound
+            } else {
+                a.read()
+            },
+            if b.is_null() {
+                Object::Unbound
+            } else {
+                b.read()
+            },
+        ]);
+        let n = 1 + usize::from(!a.is_null()) + usize::from(!b.is_null());
+        let f = &*f;
+        let r = match f.call_kw.as_ref() {
+            Some(ckw) => ckw(&views[..n], &[]),
+            None => (f.call)(&views[..n]),
+        };
+        match r {
+            Ok(v) => {
+                out.write(v);
+                0
+            }
+            Err(_) => 1,
+        }
+    }
+}
+
+unsafe extern "C" fn k_list_append(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    _: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s; growing the list runs no code.
+    unsafe {
+        let Object::List(l) = &*recv else {
+            return 1;
+        };
+        let Some(items) = l.peek_mut() else {
+            return 1;
+        };
+        crate::push_fast(items, crate::clone_hot(&*a));
+        out.write(Object::None);
+        0
+    }
+}
+
+unsafe extern "C" fn k_list_extend(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    _: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s; copying an exact list's or tuple's items runs
+    // no code.
+    unsafe {
+        let Object::List(l) = &*recv else {
+            return 1;
+        };
+        match &*a {
+            // (`xs.extend(xs)` reads the length once: it doubles.)
+            Object::List(src) if Rc::ptr_eq(l, src) => {
+                let Some(xs) = l.peek_mut() else {
+                    return 1;
+                };
+                let copy: Vec<Object> = xs.iter().map(crate::clone_hot).collect();
+                xs.extend(copy);
+            }
+            Object::List(src) => {
+                let (Some(s), Some(xs)) = (src.peek(), l.peek_mut()) else {
+                    return 1;
+                };
+                xs.extend(s.iter().map(crate::clone_hot));
+            }
+            Object::Tuple(t) => {
+                let Some(xs) = l.peek_mut() else {
+                    return 1;
+                };
+                xs.extend(t.iter().map(crate::clone_hot));
+            }
+            _ => return 1,
+        }
+        out.write(Object::None);
+        0
+    }
+}
+
+unsafe extern "C" fn k_list_pop(
+    out: *mut Object,
+    recv: *const Object,
+    _: *const Object,
+    _: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s; the item moves out.
+    unsafe {
+        let Object::List(l) = &*recv else {
+            return 1;
+        };
+        let Some(v) = l.peek_mut().and_then(Vec::pop) else {
+            return 1;
+        };
+        out.write(v);
+        0
+    }
+}
+
+unsafe extern "C" fn k_dict_get(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s; the probe runs no code.
+    unsafe {
+        let (Object::Dict(d), k) = (&*recv, &*a) else {
+            return 1;
+        };
+        let Some(probe) = crate::object::LeafProbe::new(k) else {
+            return 1;
+        };
+        let Some(m) = d.peek() else {
+            return 1;
+        };
+        let v = match m.get(&probe) {
+            Some(v) => crate::clone_hot(v),
+            None if probe.miss_is_exact() => {
+                if b.is_null() {
+                    Object::None
+                } else {
+                    crate::clone_hot(&*b)
+                }
+            }
+            None => return 1,
+        };
+        out.write(v);
+        0
+    }
+}
+
+unsafe extern "C" fn k_set_add(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    _: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s; hashing and comparing a `str`, an `int` or a
+    // tuple of them runs no code.
+    unsafe {
+        let (Object::Set(s), k) = (&*recv, &*a) else {
+            return 1;
+        };
+        if !matches!(k, Object::Str(_) | Object::Int(_) | Object::Tuple(_)) {
+            return 1;
+        }
+        let Some(probe) = crate::object::LeafProbe::new(k) else {
+            return 1;
+        };
+        let Some(m) = s.peek_mut() else {
+            return 1;
+        };
+        if m.get_index_of(&probe).is_none() {
+            // (A key of another kind that may equal it: the full call's.)
+            if !probe.miss_is_exact() {
+                return 1;
+            }
+            m.insert(crate::object::DictKey(crate::clone_hot(k)));
+        }
+        out.write(Object::None);
+        0
+    }
+}
+
+/// `set.discard` (`remove`: a miss declines, for the full call's
+/// `KeyError`) of a key that compares natively (see `LeafProbe`): the
+/// removed key is the same object, or an equal `str`, `int` or tuple of
+/// them, so its release frees nothing that runs code.
+unsafe fn set_take(out: *mut Object, recv: *const Object, a: *const Object, remove: bool) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe {
+        let (Object::Set(s), k) = (&*recv, &*a) else {
+            return 1;
+        };
+        let Some(probe) = crate::object::LeafProbe::new(k) else {
+            return 1;
+        };
+        let Some(m) = s.peek_mut() else {
+            return 1;
+        };
+        match m.swap_take(&probe) {
+            Some(key) => crate::drop_hot(key.0),
+            None if !remove && probe.miss_is_exact() => {}
+            None => return 1,
+        }
+        out.write(Object::None);
+        0
+    }
+}
+
+unsafe extern "C" fn k_set_discard(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    _: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe { set_take(out, recv, a, false) }
+}
+
+unsafe extern "C" fn k_set_remove(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    _: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe { set_take(out, recv, a, true) }
+}
+
+/// The receiver's and the first argument's text, both exact `str`s.
+///
+/// # Safety
+///
+/// As [`Kernel`]'s.
+unsafe fn two_strs<'s>(
+    recv: *const Object,
+    a: *const Object,
+) -> Option<(
+    &'s crate::shared_value::SharedStr,
+    &'s crate::shared_value::SharedStr,
+)> {
+    // SAFETY: the caller's contract.
+    match unsafe { (&*recv, &*a) } {
+        (Object::Str(s), Object::Str(p)) => Some((s, p)),
+        _ => None,
+    }
+}
+
+/// The text from character `start` (a `str` method's optional start
+/// position, null for none) of the receiver `s`, when `s` is ASCII (its
+/// byte offsets are its indices): `Some(None)` for a start past the end
+/// (the methods' "not found"), `None` to decline.
+///
+/// # Safety
+///
+/// As [`Kernel`]'s.
+unsafe fn ascii_from<'s>(
+    s: &'s crate::shared_value::SharedStr,
+    start: *const Object,
+) -> Option<Option<(&'s [u8], usize)>> {
+    let bytes = s.as_bytes();
+    if start.is_null() {
+        return Some(Some((bytes, 0)));
+    }
+    // SAFETY: the caller's contract.
+    let Object::Int(i) = (unsafe { &*start }) else {
+        return None;
+    };
+    if crate::shared_value::SharedStr::char_count(s) != bytes.len() {
+        return None;
+    }
+    let n = bytes.len() as i64;
+    let i = if *i < 0 { (*i + n).max(0) } else { *i };
+    if i > n {
+        return Some(None);
+    }
+    Some(Some((&bytes[i as usize..], i as usize)))
+}
+
+unsafe extern "C" fn k_str_startswith(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe {
+        let Some((s, p)) = two_strs(recv, a) else {
+            return 1;
+        };
+        let Some(from) = ascii_from(s, b) else {
+            return 1;
+        };
+        let hit = from.is_some_and(|(t, _)| t.starts_with(p.as_bytes()));
+        out.write(Object::Bool(hit));
+        0
+    }
+}
+
+unsafe extern "C" fn k_str_endswith(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe {
+        let Some((s, p)) = two_strs(recv, a) else {
+            return 1;
+        };
+        let Some(from) = ascii_from(s, b) else {
+            return 1;
+        };
+        let hit = from.is_some_and(|(t, _)| t.ends_with(p.as_bytes()));
+        out.write(Object::Bool(hit));
+        0
+    }
+}
+
+/// An `isdigit`-style predicate of an ASCII string: true when non-empty
+/// and every byte passes `test`. A non-ASCII string declines (unless an
+/// earlier ASCII byte already failed).
+///
+/// # Safety
+///
+/// As [`Kernel`]'s.
+unsafe fn ascii_all(out: *mut Object, recv: *const Object, test: fn(u8) -> bool) -> u32 {
+    // SAFETY: the caller's contract.
+    unsafe {
+        let Object::Str(s) = &*recv else {
+            return 1;
+        };
+        let bytes = s.as_bytes();
+        let mut all = !bytes.is_empty();
+        for &c in bytes {
+            if c >= 0x80 {
+                return 1;
+            }
+            if !test(c) {
+                all = false;
+                break;
+            }
+        }
+        out.write(Object::Bool(all));
+        0
+    }
+}
+
+unsafe extern "C" fn k_str_isdigit(
+    out: *mut Object,
+    recv: *const Object,
+    _: *const Object,
+    _: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe { ascii_all(out, recv, |c| c.is_ascii_digit()) }
+}
+
+unsafe extern "C" fn k_str_isalpha(
+    out: *mut Object,
+    recv: *const Object,
+    _: *const Object,
+    _: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe { ascii_all(out, recv, |c| c.is_ascii_alphabetic()) }
+}
+
+unsafe extern "C" fn k_str_isspace(
+    out: *mut Object,
+    recv: *const Object,
+    _: *const Object,
+    _: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe { ascii_all(out, recv, |c| crate::unicode_case::is_space(char::from(c))) }
+}
+
+unsafe extern "C" fn k_str_find(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    _: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe {
+        let Some((s, p)) = two_strs(recv, a) else {
+            return 1;
+        };
+        // (An ASCII receiver's byte offsets are its indices.)
+        if crate::shared_value::SharedStr::char_count(s) != s.len() {
+            return 1;
+        }
+        let Some(from) = ascii_from(s, b) else {
+            return 1;
+        };
+        let at = from
+            .and_then(|(_, i)| crate::builtins::substr_find(&s[i..], p).map(|k| (k + i) as i64));
+        out.write(Object::Int(at.unwrap_or(-1)));
+        0
+    }
+}
+
+/// A `str` method whose body runs no Python code for any arguments
+/// (`lower`, `upper`, the registered pure bodies).
+unsafe extern "C" fn k_str_body(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    f: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe { run_body(out, recv, a, b, f) }
+}
+
+unsafe extern "C" fn k_str_strip(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    f: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe {
+        if !a.is_null() && !matches!(&*a, Object::Str(_) | Object::None) {
+            return 1;
+        }
+        run_body(out, recv, a, b, f)
+    }
+}
+
+unsafe extern "C" fn k_str_split(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    f: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe {
+        if !a.is_null() && !matches!(&*a, Object::Str(_) | Object::None) {
+            return 1;
+        }
+        run_body(out, recv, a, b, f)
+    }
+}
+
+unsafe extern "C" fn k_str_join(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    f: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s; an exact list's or tuple's exact strings
+    // join with no iteration hook.
+    unsafe {
+        let strs = match &*a {
+            Object::List(l) => l
+                .peek()
+                .is_some_and(|xs| xs.iter().all(|x| matches!(x, Object::Str(_)))),
+            Object::Tuple(t) => t.iter().all(|x| matches!(x, Object::Str(_))),
+            _ => false,
+        };
+        if !strs {
+            return 1;
+        }
+        run_body(out, recv, a, b, f)
+    }
+}
+
+unsafe extern "C" fn k_str_replace(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    f: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe {
+        if !matches!((&*a, &*b), (Object::Str(_), Object::Str(_))) {
+            return 1;
+        }
+        run_body(out, recv, a, b, f)
+    }
+}
+
 /// [`h_container`]'s raise: the error in `st`, `st.len` and `st.pc` past
 /// the instruction.
 const CONTAINER_RAISED: u64 = u64::MAX - 1;
@@ -1443,43 +2130,418 @@ unsafe extern "C" fn h_container(
             crate::drop_hot(st.stack.add(len - 3).read());
             return (len - 3) as u64;
         }
-        // An instance whose class's `__getitem__` is a native fast
-        // subscript the site cached.
-        if ins.op != OpCode::BinarySubscr || len < 2 {
-            return u64::MAX;
+        // An instance whose class's `__getitem__` is a native fast
+        // subscript the site cached.
+        if ins.op != OpCode::BinarySubscr || len < 2 {
+            return u64::MAX;
+        }
+        let ops = std::slice::from_raw_parts(st.stack.add(len - 2), 2);
+        let Some(fast) = (*st.interp).core_native_subscript(&*code, pc as usize, ops) else {
+            return u64::MAX;
+        };
+        // (An error leaves the subscript to the core loop, which raises it.)
+        let Some(Ok(v)) = fast(ops) else {
+            return u64::MAX;
+        };
+        #[cfg(test)]
+        crate::NATIVE_FAST_SUBSCRIPTS.with(|calls| calls.set(calls.get() + 1));
+        crate::drop_hot(st.stack.add(len - 1).read());
+        crate::drop_hot(st.stack.add(len - 2).read());
+        st.stack.add(len - 2).write(v);
+        (len - 1) as u64
+    }
+}
+
+/// `CONTAINS_OP` of the item at `at` and the container above it: `0` or
+/// `1` (found; both left the stack), anything else declined untouched.
+unsafe extern "C" fn h_contains(at: *mut Object) -> u32 {
+    // SAFETY: the code passes two initialized stack slots.
+    unsafe {
+        let (item, container) = (&*at, &*at.add(1));
+        if !droppable(item) || !droppable(container) {
+            return 2;
+        }
+        let Some(found) = Interpreter::leaf_contains(container, item) else {
+            return 2;
+        };
+        crate::drop_hot(at.add(1).read());
+        crate::drop_hot(at.read());
+        u32::from(found)
+    }
+}
+
+/// `CONTAINS_OP` of the item at `item` in the container at `cont`, each
+/// read in place: a local's or constant's (borrowed), or a stack slot's
+/// (owned there, its bit in `owned`: `1` the item, `2` the container),
+/// which leaves the stack. `0` or `1` (found), anything else declined
+/// untouched.
+unsafe fn contains_fast(item: *const Object, cont: *const Object, owned: u32) -> u32 {
+    // SAFETY: the code passes two initialized values, the owned ones in
+    // stack slots.
+    unsafe {
+        let (i, c) = (&*item, &*cont);
+        if owned != 0 && !(droppable(i) && droppable(c)) {
+            return 2;
+        }
+        let Some(found) = Interpreter::leaf_contains(c, i) else {
+            return 2;
+        };
+        if owned & 2 != 0 {
+            crate::drop_hot(cont.read());
+        }
+        if owned & 1 != 0 {
+            crate::drop_hot(item.read());
+        }
+        u32::from(found)
+    }
+}
+
+/// `BINARY_SUBSCR` of the container at `cont` by the key at `key`, each
+/// read in place as [`contains_fast`] reads its operands (`owned`: `1`
+/// the container, `2` the key): an exact `dict` by a key that compares
+/// natively, a `list` or `tuple` by an in-range `int`, or a `str` by an
+/// in-range `int` (an ASCII one's). `0` the item written to `out` (the
+/// owned operands released first), anything else declined untouched.
+unsafe fn subscr_fast(
+    out: *mut Object,
+    cont: *const Object,
+    key: *const Object,
+    owned: u32,
+) -> u32 {
+    // SAFETY: the code passes two initialized values, the owned ones in
+    // stack slots, and a free slot (or the container's own) for the item.
+    unsafe {
+        let (c, k) = (&*cont, &*key);
+        if owned != 0 && !(droppable(c) && droppable(k)) {
+            return 1;
+        }
+        let index = |n: usize| -> Option<usize> {
+            let Object::Int(i) = *k else {
+                return None;
+            };
+            let i = if i < 0 { i.checked_add(n as i64)? } else { i };
+            usize::try_from(i).ok().filter(|&i| i < n)
+        };
+        let r = match c {
+            Object::Dict(d) => {
+                let Some(probe) = crate::object::LeafProbe::new(k) else {
+                    return 1;
+                };
+                let Some(v) = d.peek().and_then(|m| m.get(&probe)) else {
+                    return 1;
+                };
+                crate::clone_hot(v)
+            }
+            Object::List(xs) => {
+                let Some(xs) = xs.peek() else {
+                    return 1;
+                };
+                let Some(i) = index(xs.len()) else {
+                    return 1;
+                };
+                crate::clone_hot(&xs[i])
+            }
+            Object::Tuple(t) => {
+                let Some(i) = index(t.len()) else {
+                    return 1;
+                };
+                crate::clone_hot(&t[i])
+            }
+            Object::Str(s) => {
+                let bytes = s.as_bytes();
+                if crate::shared_value::SharedStr::char_count(s) != bytes.len() {
+                    return 1;
+                }
+                let Some(i) = index(bytes.len()) else {
+                    return 1;
+                };
+                Object::from_char(char::from(bytes[i]))
+            }
+            _ => return 1,
+        };
+        if owned & 2 != 0 {
+            crate::drop_hot(key.read());
+        }
+        if owned & 1 != 0 {
+            crate::drop_hot(cont.read());
+        }
+        out.write(r);
+        0
+    }
+}
+
+/// `UNPACK_SEQUENCE n` of the exact `n`-item tuple or list at `src`, read
+/// in place (a local's or constant's), or owned in the stack slot `dst`
+/// itself (`owned`), which leaves the stack: its items to `dst` and up,
+/// the last item first, as the core loop's arm pushes them. `0` unpacked,
+/// anything else declined untouched.
+unsafe fn unpack_fast(
+    dst: *mut Object,
+    src: *const Object,
+    n: u64,
+    owned: u32,
+    interp: &Interpreter,
+) -> u32 {
+    // SAFETY: the code passes an initialized value (in `dst` itself when
+    // owned) and `n` free slots from `dst` up (the first the owned
+    // value's).
+    unsafe {
+        let n = n as usize;
+        let s = &*src;
+        let fits = match s {
+            Object::Tuple(t) => t.len() == n,
+            Object::List(l) => l.peek().is_some_and(|xs| xs.len() == n),
+            _ => false,
+        };
+        // An owned sequence's release frees nothing that could finalize
+        // when this is its only reference (its items outlive it on the
+        // stack); a shared one's is a plain decrement.
+        let unique = match s {
+            Object::Tuple(t) => crate::shared_value::ThinArc::strong_count(t) == 1,
+            Object::List(l) => Rc::strong_count(l) == 1,
+            _ => false,
+        };
+        if !fits || (owned != 0 && !unique && !droppable(s)) {
+            return 1;
+        }
+        let seq = if owned != 0 { Some(src.read()) } else { None };
+        let s = seq.as_ref().unwrap_or(s);
+        match s {
+            Object::Tuple(t) => {
+                for (k, item) in t.iter().rev().enumerate() {
+                    dst.add(k).write(crate::clone_hot(item));
+                }
+            }
+            Object::List(l) => {
+                let xs = l.peek().expect("checked above");
+                for (k, item) in xs.iter().rev().enumerate() {
+                    dst.add(k).write(crate::clone_hot(item));
+                }
+            }
+            _ => unreachable!("checked above"),
+        }
+        // (A dead tuple's allocation is recycled, as `release` does it.)
+        match seq {
+            Some(seq @ Object::Tuple(_)) => interp.maybe_donate_tuple(seq),
+            Some(seq) => crate::drop_hot(seq),
+            None => {}
+        }
+        0
+    }
+}
+
+/// `STORE_SUBSCR` of the value at `val` into the container at `cont` by
+/// the key at `key`, each read in place as [`contains_fast`] reads its
+/// operands (`owned`: `1` the value, `2` the container, `4` the key): an
+/// exact `list` by an in-range `int`, or an exact `dict` by a key that
+/// compares natively, as the core loop's arm stores. `0` stored (an owned
+/// value moved in, a borrowed one copied; the other owned operands and
+/// the displaced value released), anything else declined untouched.
+unsafe fn store_subscr_fast(
+    val: *const Object,
+    cont: *const Object,
+    key: *const Object,
+    owned: u32,
+) -> u32 {
+    // SAFETY: the code passes three initialized values, the owned ones in
+    // stack slots. The store runs no code; a displaced value is released
+    // after it (a finalizer it queues runs before the next instruction).
+    unsafe {
+        let (c, k) = (&*cont, &*key);
+        if !Interpreter::core_droppable(c) || (owned & 4 != 0 && !droppable(k)) {
+            return 1;
+        }
+        let displaced_ok = |old: &Object| {
+            matches!(
+                old,
+                Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None
+            ) || Interpreter::core_droppable(old)
+        };
+        let value = || {
+            if owned & 1 != 0 {
+                val.read()
+            } else {
+                crate::clone_hot(&*val)
+            }
+        };
+        let old = match (c, k) {
+            (Object::List(xs), Object::Int(i)) => {
+                let Some(xs) = xs.peek_mut() else {
+                    return 1;
+                };
+                let n = xs.len() as i64;
+                let i = if *i < 0 { *i + n } else { *i };
+                if i < 0 || i >= n || !displaced_ok(&xs[i as usize]) {
+                    return 1;
+                }
+                std::mem::replace(&mut xs[i as usize], value())
+            }
+            (Object::Dict(cell), Object::Str(_) | Object::Int(_) | Object::Tuple(_)) => {
+                if crate::capi_watchers::dicts_active() {
+                    return 1;
+                }
+                let Some(probe) = crate::object::LeafProbe::new(k) else {
+                    return 1;
+                };
+                let Some(d) = cell.peek_mut() else {
+                    return 1;
+                };
+                let (old, changed) = match d.get_mut(&probe) {
+                    Some(slot) => {
+                        if !displaced_ok(slot) {
+                            return 1;
+                        }
+                        let old = std::mem::replace(slot, value());
+                        let changed = !old.is_same(slot);
+                        (old, changed)
+                    }
+                    None => {
+                        if !probe.miss_is_exact() {
+                            return 1;
+                        }
+                        d.insert(crate::object::DictKey(crate::clone_hot(k)), value());
+                        (Object::None, true)
+                    }
+                };
+                if changed {
+                    crate::object::dict_mutation_event(cell);
+                }
+                old
+            }
+            _ => return 1,
+        };
+        if owned & 4 != 0 {
+            crate::drop_hot(key.read());
+        }
+        if owned & 2 != 0 {
+            crate::drop_hot(cont.read());
+        }
+        crate::drop_hot(old);
+        0
+    }
+}
+
+/// Copy the borrowed operands among `ops` (bit `k` of `owned` clear) to
+/// the free stack slots `at.add(k)`, so that all of them are on the
+/// stack, as the container helpers take them.
+///
+/// # Safety
+///
+/// Each pointer addresses an initialized value; the slots of the
+/// borrowed ones are free.
+unsafe fn own_operands(at: *mut Object, ops: &[*const Object], owned: u32) {
+    for (k, &p) in ops.iter().enumerate() {
+        if owned & (1 << k) == 0 {
+            // SAFETY: the caller's contract.
+            unsafe { at.add(k).write(crate::clone_hot(&*p)) };
+        }
+    }
+}
+
+/// `CONTAINS_OP` of a local's, constant's or stack slot's item and
+/// container (bit `k` of `owned`: the one owned in the slot `at.add(k)`),
+/// read in place where they lie (see [`contains_fast`]); any other shape
+/// with both operands copied to the stack and through [`h_contains`],
+/// whose result it is.
+unsafe extern "C" fn h_contains_ref(
+    item: *const Object,
+    cont: *const Object,
+    owned: u32,
+    at: *mut Object,
+) -> u32 {
+    // SAFETY: the code passes the operands, and the two slots from `at`
+    // (the owned operands' own).
+    unsafe {
+        let r = contains_fast(item, cont, owned);
+        if r <= 1 {
+            return r;
         }
-        let ops = std::slice::from_raw_parts(st.stack.add(len - 2), 2);
-        let Some(fast) = (*st.interp).core_native_subscript(&*code, pc as usize, ops) else {
-            return u64::MAX;
-        };
-        // (An error leaves the subscript to the core loop, which raises it.)
-        let Some(Ok(v)) = fast(ops) else {
-            return u64::MAX;
-        };
-        #[cfg(test)]
-        crate::NATIVE_FAST_SUBSCRIPTS.with(|calls| calls.set(calls.get() + 1));
-        crate::drop_hot(st.stack.add(len - 1).read());
-        crate::drop_hot(st.stack.add(len - 2).read());
-        st.stack.add(len - 2).write(v);
-        (len - 1) as u64
+        own_operands(at, &[item, cont], owned);
+        h_contains(at)
     }
 }
 
-/// `CONTAINS_OP` of the item at `at` and the container above it: `0` or
-/// `1` (found; both left the stack), anything else declined untouched.
-unsafe extern "C" fn h_contains(at: *mut Object) -> u32 {
-    // SAFETY: the code passes two initialized stack slots.
+/// `BINARY_SUBSCR` (the `pc`th instruction `ins` of `code`) on a stack
+/// `len` deep whose top two slots are the container's and key's (each
+/// owned there, by the bits of `flags`, or a local's or constant's at
+/// `cont` / `key`): through [`subscr_fast`], else with both operands on
+/// the stack through [`h_container`] (`flags` bit 8: it may raise), whose
+/// result it is.
+#[allow(clippy::too_many_arguments)]
+unsafe extern "C" fn h_subscr_ref(
+    st: *mut State,
+    ins: *const weavepy_compiler::Instruction,
+    code: *const CodeObject,
+    pc: u64,
+    len: u64,
+    cont: *const Object,
+    key: *const Object,
+    flags: u64,
+) -> u64 {
+    // SAFETY: the code passes its live state, one of its own instructions
+    // and its code, its depth, and the operands.
     unsafe {
-        let (item, container) = (&*at, &*at.add(1));
-        if !droppable(item) || !droppable(container) {
-            return 2;
+        let at = (*st).stack.add(len as usize - 2);
+        let owned = (flags & 0xff) as u32;
+        if subscr_fast(at, cont, key, owned) == 0 {
+            return len - 1;
         }
-        let Some(found) = Interpreter::leaf_contains(container, item) else {
-            return 2;
-        };
-        crate::drop_hot(at.add(1).read());
-        crate::drop_hot(at.read());
-        u32::from(found)
+        own_operands(at, &[cont, key], owned);
+        h_container(st, ins, code, pc, len, flags >> 8)
+    }
+}
+
+/// `UNPACK_SEQUENCE n` as [`h_subscr_ref`] runs a subscript: through
+/// [`unpack_fast`] on the sequence in the top slot or at `src`, else on
+/// the stack through [`h_container`].
+#[allow(clippy::too_many_arguments)]
+unsafe extern "C" fn h_unpack_ref(
+    st: *mut State,
+    ins: *const weavepy_compiler::Instruction,
+    code: *const CodeObject,
+    pc: u64,
+    len: u64,
+    src: *const Object,
+    n: u64,
+    flags: u64,
+) -> u64 {
+    // SAFETY: as `h_subscr_ref`'s.
+    unsafe {
+        let at = (*st).stack.add(len as usize - 1);
+        let owned = (flags & 0xff) as u32;
+        if unpack_fast(at, src, n, owned, &*(*st).interp) == 0 {
+            return len - 1 + n;
+        }
+        own_operands(at, &[src], owned);
+        h_container(st, ins, code, pc, len, flags >> 8)
+    }
+}
+
+/// `STORE_SUBSCR` as [`h_subscr_ref`] runs a subscript: through
+/// [`store_subscr_fast`] on the value, container and key in the top
+/// three slots or at `val`, `cont`, `key`, else on the stack through
+/// [`h_container`].
+#[allow(clippy::too_many_arguments)]
+unsafe extern "C" fn h_store_subscr_ref(
+    st: *mut State,
+    ins: *const weavepy_compiler::Instruction,
+    code: *const CodeObject,
+    pc: u64,
+    len: u64,
+    val: *const Object,
+    cont: *const Object,
+    key: *const Object,
+    flags: u64,
+) -> u64 {
+    // SAFETY: as `h_subscr_ref`'s.
+    unsafe {
+        let at = (*st).stack.add(len as usize - 3);
+        let owned = (flags & 0xff) as u32;
+        if store_subscr_fast(val, cont, key, owned) == 0 {
+            return len - 3;
+        }
+        own_operands(at, &[val, cont, key], owned);
+        h_container(st, ins, code, pc, len, flags >> 8)
     }
 }
 
@@ -2028,6 +3090,7 @@ unsafe extern "C" fn h_load_method(
     pc: u64,
     len: u64,
     cache: *mut MethodCache,
+    mcache: *const ModCache,
 ) -> u32 {
     // SAFETY: the code passes its live state, its own code and extension,
     // one of its method loads, and its stack depth (at least one, with
@@ -2049,8 +3112,24 @@ unsafe extern "C" fn h_load_method(
         // `module.func(...)`: the attribute with an empty self slot, as the
         // core loop's arm loads it.
         if let Object::Module(m) = &*top {
-            let Some(v) = Interpreter::core_module_attr(code, m, pc, name) else {
-                return 1;
+            // The entry the site last read, while the module's dict is
+            // exactly as it was then (its stamp moves with every change).
+            let mc = &*mcache;
+            let hit = m.dict.peek().and_then(|d| {
+                (std::ptr::eq(Rc::as_ptr(&m.dict).cast::<u8>(), mc.dict.get() as *const u8)
+                    && d.mutation_stamp() == mc.stamp.get())
+                .then(|| d.get_index(mc.idx.get()).map(|(_, v)| crate::clone_hot(v)))
+                .flatten()
+            });
+            let v = match hit {
+                Some(v) => v,
+                None => {
+                    let Some(v) = Interpreter::core_module_attr(code, m, pc, name) else {
+                        return 1;
+                    };
+                    mc.note(code, m, pc);
+                    v
+                }
             };
             crate::drop_hot(std::mem::replace(&mut *top, v));
             st.stack.add(len).write(Object::Unbound);
@@ -2186,6 +3265,33 @@ fn fill_method_cache(
     };
 }
 
+/// A method load's module-attribute shortcut: the module dict's address
+/// and stamp when the site last read it, and the entry's position (`0`
+/// address: empty).
+#[derive(Default)]
+pub(crate) struct ModCache {
+    dict: std::cell::Cell<usize>,
+    stamp: std::cell::Cell<u64>,
+    idx: std::cell::Cell<usize>,
+}
+
+impl ModCache {
+    /// Remember the entry the site's inline cache just read from `m`.
+    #[cold]
+    fn note(&self, code: &CodeObject, m: &crate::object::PyModule, pc: usize) {
+        use weavepy_compiler::InlineCache as IC;
+        let IC::LoadAttrModule { key_idx, .. } = code.caches.get(pc as u32) else {
+            return;
+        };
+        // SAFETY: a read between two instructions.
+        if let Some(d) = unsafe { m.dict.peek() } {
+            self.dict.set(Rc::as_ptr(&m.dict) as *const u8 as usize);
+            self.stamp.set(d.mutation_stamp());
+            self.idx.set(key_idx as usize);
+        }
+    }
+}
+
 /// A `LOAD_ATTR` site's `__slots__` member shortcut: the class version and
 /// the member's position the site's cache last proved (`ver` 0 empty),
 /// read without decoding the site's cache.
@@ -3432,6 +4538,110 @@ unsafe extern "C" fn h_call(
     }
 }
 
+/// `len(x)` of an exact container or string.
+fn len_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
+    // SAFETY: reads between two instructions (the lengths run no code).
+    let n = unsafe {
+        match args {
+            [Object::List(l)] => l.peek()?.len(),
+            [Object::Tuple(t)] => t.len(),
+            [Object::Str(s)] => crate::shared_value::SharedStr::char_count(s),
+            [Object::Dict(d)] => d.peek()?.len(),
+            [Object::Set(s)] => s.peek()?.len(),
+            [Object::Bytes(b)] => b.len(),
+            _ => return None,
+        }
+    };
+    Some(Ok(Object::Int(n as i64)))
+}
+
+/// [`h_call`] at a site settled on a leaf builtin: a container's or
+/// string's method through its [`Kernel`], or `isinstance`, `len` or a
+/// registered fast half in place as `h_call` runs them first, from a
+/// small function (the general helper stays as it is); anything else
+/// (another callee, a declined shape) through `h_call`.
+unsafe extern "C" fn h_call_leaf(
+    st: *mut State,
+    code: *const CodeObject,
+    ext: *const CodeConstObjects,
+    pc: u64,
+    len: u64,
+) -> u32 {
+    use crate::LeafKind;
+    // SAFETY: as `h_call`'s.
+    unsafe {
+        let s = &mut *st;
+        let (c, x, pcu, lenu) = (&*code, &*ext, pc as usize, len as usize);
+        let argc = c.instructions[pcu].arg as usize;
+        if lenu >= argc + 2 && !s.sw.is_null() {
+            let base = s.stack;
+            let start = lenu - argc - 2;
+            if let Object::Builtin(b) = &*base.add(start) {
+                let kind = x
+                    .method_slots
+                    .get()
+                    .and_then(|m| m.get(pcu))
+                    .and_then(|m| m.get_leaf(b));
+                // A container's or string's method with a kernel, on the
+                // receiver in the self slot.
+                if let (Some(k), false) = (kind, matches!(&*base.add(start + 1), Object::Unbound)) {
+                    let args = std::slice::from_raw_parts(base.add(start + 1), argc + 1);
+                    if Rc::strong_count(b) > 1 && Interpreter::core_all_droppable(args) {
+                        if let Some(v) = kernel_call(k, args, b) {
+                            for j in start..lenu {
+                                crate::drop_hot(base.add(j).read());
+                            }
+                            base.add(start).write(v);
+                            s.len = start + 1;
+                            return 0;
+                        }
+                    }
+                }
+                if matches!(
+                    kind,
+                    Some(LeafKind::Isinstance | LeafKind::Len | LeafKind::Fast(_))
+                ) && Rc::strong_count(b) > 1
+                {
+                    let first = if matches!(&*base.add(start + 1), Object::Unbound) {
+                        start + 2
+                    } else {
+                        start + 1
+                    };
+                    let args = std::slice::from_raw_parts(base.add(first), lenu - first);
+                    let r = if Interpreter::core_all_droppable(args) {
+                        match kind {
+                            Some(LeafKind::Fast(f)) => f(args),
+                            Some(LeafKind::Len) => len_fast(args),
+                            _ => Interpreter::core_isinstance(args),
+                        }
+                    } else {
+                        None
+                    };
+                    if let Some(r) = r {
+                        for k in start..lenu {
+                            crate::drop_hot(base.add(k).read());
+                        }
+                        return match r {
+                            Ok(v) => {
+                                base.add(start).write(v);
+                                s.len = start + 1;
+                                0
+                            }
+                            Err(e) => {
+                                s.len = start;
+                                s.pc = pcu + 1;
+                                s.err = Some(e);
+                                RAISED
+                            }
+                        };
+                    }
+                }
+            }
+        }
+        h_call(st, code, ext, pc, len)
+    }
+}
+
 /// The `CALL` at `pc` (its operands from `start` up a `len`-deep stack) of a
 /// leaf builtin whose leaf half declined, or of an instance's
 /// `__reduce_ex__`, through the core loop's lane for them
@@ -4161,6 +5371,8 @@ fn compile_with(
     let mut slots: Vec<Box<SlotCache>>;
     let mut methods: Vec<Box<MethodCache>>;
     let mut direct: Vec<Box<DirectSite>>;
+    let mut held: Vec<Rc<crate::object::BuiltinFn>>;
+    let mut mods: Vec<Box<ModCache>>;
     let t0 = stats::enabled().then(std::time::Instant::now);
     // A body whose code comes out large is lowered again lean: compiling
     // costs about the same per IR instruction, and a large body's time
@@ -4176,6 +5388,8 @@ fn compile_with(
         slots = std::mem::take(&mut lower.slot_caches);
         methods = std::mem::take(&mut lower.method_caches);
         direct = std::mem::take(&mut lower.direct_sites);
+        held = std::mem::take(&mut lower.held);
+        mods = std::mem::take(&mut lower.mod_caches);
         if done {
             lower.b.seal_all_blocks();
             lower.b.finalize();
@@ -4237,6 +5451,8 @@ fn compile_with(
         _slots: slots,
         _methods: methods,
         _direct: direct,
+        _held: held,
+        _mods: mods,
     })
 }
 
@@ -4340,6 +5556,11 @@ struct Lower<'a> {
     method_caches: Vec<Box<MethodCache>>,
     #[allow(clippy::vec_box)]
     direct_sites: Vec<Box<DirectSite>>,
+    /// The builtins the method kernels' calls address.
+    held: Vec<Rc<crate::object::BuiltinFn>>,
+    /// The method loads' module-attribute caches.
+    #[allow(clippy::vec_box)]
+    mod_caches: Vec<Box<ModCache>>,
     /// The block a helper's raise goes to (see [`Self::leave`]), made on
     /// first use.
     catch: Option<Block>,
@@ -4504,6 +5725,8 @@ impl<'a> Lower<'a> {
             slot_caches: Vec::new(),
             method_caches: Vec::new(),
             direct_sites: Vec::new(),
+            held: Vec::new(),
+            mod_caches: Vec::new(),
             catch: None,
         }
     }
@@ -4692,19 +5915,22 @@ impl<'a> Lower<'a> {
         let by_value = self.b.create_block();
         let by_clone = self.b.create_block();
         let done = self.b.create_block();
-        if self.tags.rc_counted == 0 {
+        if self.tags.rc_counted == 0 && self.tags.rc_counted_hdr == 0 {
             self.b.ins().brif(scalar, by_value, &[], by_clone, &[]);
         } else {
-            // An instance, list, dict or class while one thread owns every
-            // count: a plain increment of its count.
+            // An instance, list, dict, class, string or tuple while one
+            // thread owns every count: a plain increment of its count.
             let counted = self.b.create_block();
             let heap = self.b.create_block();
             self.b.ins().brif(scalar, by_value, &[], heap, &[]);
             self.b.switch_to_block(heap);
             let one = self.b.ins().iconst(types::I64, 1);
             let bit = self.b.ins().ishl(one, tag);
-            let rc = self.b.ins().band_imm(bit, self.tags.rc_counted);
-            let rc = self.b.ins().icmp_imm(IntCC::NotEqual, rc, 0);
+            let any = self
+                .b
+                .ins()
+                .band_imm(bit, self.tags.rc_counted | self.tags.rc_counted_hdr);
+            let rc = self.b.ins().icmp_imm(IntCC::NotEqual, any, 0);
             let flag = self.b.ins().iconst(
                 self.ptr,
                 std::ptr::from_ref(crate::sync::rc_shared_flag()) as i64,
@@ -4714,10 +5940,10 @@ impl<'a> Lower<'a> {
             let ok = self.b.ins().band(rc, solo);
             self.b.ins().brif(ok, counted, &[], by_clone, &[]);
             self.b.switch_to_block(counted);
-            let w = self.b.ins().load(self.ptr, FLAGS, src, 8);
-            let c = self.b.ins().load(types::I64, FLAGS, w, 0);
+            let at = self.count_addr(src, bit);
+            let c = self.b.ins().load(types::I64, FLAGS, at, 0);
             let c1 = self.b.ins().iadd_imm(c, 1);
-            self.b.ins().store(FLAGS, c1, w, 0);
+            self.b.ins().store(FLAGS, c1, at, 0);
             self.b.ins().jump(by_value, &[]);
         }
         self.b.switch_to_block(by_value);
@@ -4729,6 +5955,20 @@ impl<'a> Lower<'a> {
         self.b.switch_to_block(done);
     }
 
+    /// The strong count's address of the counted heap value at `src`
+    /// whose tag bit is `bit`: its payload word, or two words before it
+    /// for a [`Tags::rc_counted_hdr`] variant.
+    fn count_addr(&mut self, src: Value, bit: Value) -> Value {
+        let w = self.b.ins().load(self.ptr, FLAGS, src, 8);
+        if self.tags.rc_counted_hdr == 0 {
+            return w;
+        }
+        let hdr = self.b.ins().band_imm(bit, self.tags.rc_counted_hdr);
+        let hdr = self.b.ins().icmp_imm(IntCC::NotEqual, hdr, 0);
+        let before = self.b.ins().iadd_imm(w, -16);
+        self.b.ins().select(hdr, before, w)
+    }
+
     /// Write the whole virtual stack to the frame's.
     fn flush(&mut self) {
         let base = self.depth - self.vs.len();
@@ -5217,7 +6457,8 @@ impl<'a> Lower<'a> {
                         let scalar = self.is_scalar(tag);
                         let heap = self.b.create_block();
                         let done = self.b.create_block();
-                        self.b.ins().brif(scalar, done, &[], heap, &[]);
+                        // (A shared counted value by a plain decrement.)
+                        self.release_or(scalar, at, tag, done, heap);
                         self.b.switch_to_block(heap);
                         let r = self
                             .call(h_pop as *const () as usize, &[at], true)
@@ -5385,6 +6626,9 @@ impl<'a> Lower<'a> {
             }
             OpCode::ForIter => return self.for_iter(pc, ins.arg),
             OpCode::BinarySubscr if self.seq_index(pc) => true,
+            OpCode::BinarySubscr if self.subscr_ref(pc) => true,
+            OpCode::UnpackSequence if self.unpack_ref(pc, ins.arg as usize) => true,
+            OpCode::StoreSubscr if self.store_subscr_ref(pc) => true,
             OpCode::BinarySubscr
             | OpCode::BinarySlice
             | OpCode::StoreSubscr
@@ -5741,6 +6985,7 @@ impl<'a> Lower<'a> {
             Some(OpCode::LoadMethodAttr)
                 if !self.is_gen() && crate::method_site_in_place(self.ext, pc + 1) =>
             {
+                self.method_kernel(pc);
                 self.method_call(pc)
             }
             Some(OpCode::StoreAttr) => return self.store_attr(pc),
@@ -5771,6 +7016,118 @@ impl<'a> Lower<'a> {
         self.code.is_generator || self.code.is_coroutine || self.code.is_async_generator
     }
 
+    /// `LOAD_FAST x; x.m(<up to two local, constant or small-int
+    /// arguments>); CALL` on an exact `list`, `dict`, `set` or `str` whose
+    /// method the site settled as a leaf with a [`Kernel`]: the receiver's
+    /// tag checked in line and the kernel called on the operands in place
+    /// (no method object, receiver copy or argument copies), continuing
+    /// after the `CALL`. Lowering continues (at the general fused call)
+    /// where the receiver is something else or the kernel declines;
+    /// nothing is emitted for a site without a kernel.
+    fn method_kernel(&mut self, pc: usize) {
+        #[derive(Clone, Copy)]
+        enum Arg {
+            Local(u32),
+            Const(u32),
+            Small(u32),
+        }
+        let code = self.code;
+        let ins = &code.instructions;
+        let n = ins.len();
+        let mut args: Vec<Arg> = Vec::new();
+        let mut k = pc + 2;
+        let call_pc = loop {
+            let Some(i) = ins.get(k) else {
+                return;
+            };
+            match i.op {
+                OpCode::LoadFast | OpCode::LoadFastBorrow if (i.arg as usize) < self.nlocals => {
+                    args.push(Arg::Local(i.arg));
+                }
+                OpCode::LoadConst if (i.arg as usize) < self.ext.objects.len() => {
+                    args.push(Arg::Const(i.arg));
+                }
+                OpCode::LoadSmallInt => args.push(Arg::Small(i.arg)),
+                OpCode::Call if i.arg as usize == args.len() => break k,
+                _ => return,
+            }
+            if args.len() > 2 {
+                return;
+            }
+            k += 1;
+        };
+        if call_pc + 1 >= n || self.blocks.get(call_pc + 1).copied().flatten().is_none() {
+            return;
+        }
+        let Some((tag, kind, f)) = crate::builtin_method_site(self.ext, pc + 1, call_pc) else {
+            return;
+        };
+        let Some(kernel) = kernel_for(tag, kind, args.len()) else {
+            return;
+        };
+        let want = match tag {
+            1 => self.tags.list,
+            2 => self.tags.dict,
+            3 => self.tags.set,
+            _ => self.tags.str,
+        };
+        self.flush();
+        let slot = self.depth;
+        let declined = self.b.create_block();
+        let recv = self.local_addr(self.code.instructions[pc].arg);
+        let t = self.tag_at(recv);
+        let other = self.b.ins().icmp_imm(IntCC::NotEqual, t, i64::from(want));
+        self.branch_out(other, declined);
+        let mut addrs = Vec::with_capacity(2);
+        for (j, arg) in args.iter().enumerate() {
+            let at = match *arg {
+                Arg::Local(i) => {
+                    let at = self.local_addr(i);
+                    if self.maybe_unbound[i as usize] && !self.bound[i as usize] {
+                        let t = self.tag_at(at);
+                        let unbound =
+                            self.b
+                                .ins()
+                                .icmp_imm(IntCC::Equal, t, i64::from(self.tags.unbound));
+                        self.branch_out(unbound, declined);
+                    }
+                    at
+                }
+                Arg::Const(c) => self.const_addr(c),
+                // A scalar in the slot the argument would take.
+                Arg::Small(v) => {
+                    let at = self.slot_addr(slot + 2 + j);
+                    self.write_tag(at, self.tags.int);
+                    let v = self.b.ins().iconst(types::I64, i64::from(v));
+                    self.b.ins().store(FLAGS, v, at, 8);
+                    at
+                }
+            };
+            addrs.push(at);
+        }
+        let null = self.b.ins().iconst(self.ptr, 0);
+        let a = addrs.first().copied().unwrap_or(null);
+        let b = addrs.get(1).copied().unwrap_or(null);
+        let fp = self.b.ins().iconst(self.ptr, Rc::as_ptr(&f) as i64);
+        self.held.push(f);
+        let out = self.slot_addr(slot);
+        let r = self
+            .call(kernel as *const () as usize, &[out, recv, a, b, fp], true)
+            .expect("returns");
+        let ran = self.b.create_block();
+        self.b.ins().brif(r, declined, &[], ran, &[]);
+        // Ran: the result in the receiver's slot, as after the `CALL`.
+        self.b.switch_to_block(ran);
+        let (depth, last) = (self.depth, self.last_pc);
+        self.push(Item::Mem(slot));
+        self.set_last(call_pc);
+        self.goto(call_pc + 1);
+        self.depth = depth;
+        self.last_pc = last;
+        self.vs.clear();
+        self.b.switch_to_block(declined);
+    }
+
     /// `LOAD_FAST x; x.m(<simple arguments>)` through [`h_local_method`],
     /// continuing at whatever follows the call (through the dispatch).
     /// Lowering continues where the helper declines (the plain load).
@@ -5978,7 +7335,7 @@ impl<'a> Lower<'a> {
     /// a plain decrement (one thread owns every count, and no watcher or
     /// tracer wants the release). Anything else goes to `slow`.
     fn release_or(&mut self, plain: Value, local: Value, old: Value, fast: Block, slow: Block) {
-        if self.cold || self.tags.rc_counted == 0 {
+        if self.cold || (self.tags.rc_counted == 0 && self.tags.rc_counted_hdr == 0) {
             self.b.ins().brif(plain, fast, &[], slow, &[]);
             return;
         }
@@ -5989,7 +7346,10 @@ impl<'a> Lower<'a> {
         self.b.switch_to_block(heap);
         let one = self.b.ins().iconst(types::I64, 1);
         let bit = self.b.ins().ishl(one, old);
-        let rc = self.b.ins().band_imm(bit, self.tags.rc_counted);
+        let rc = self
+            .b
+            .ins()
+            .band_imm(bit, self.tags.rc_counted | self.tags.rc_counted_hdr);
         let rc = self.b.ins().icmp_imm(IntCC::NotEqual, rc, 0);
         // The single-thread bias, and the release observers, all off.
         let mut quiet = rc;
@@ -6001,7 +7361,7 @@ impl<'a> Lower<'a> {
         }
         self.b.ins().brif(quiet, counted, &[], slow, &[]);
         self.b.switch_to_block(counted);
-        let w = self.b.ins().load(self.ptr, FLAGS, local, 8);
+        let w = self.count_addr(local, bit);
         let c = self.b.ins().load(types::I64, FLAGS, w, 0);
         let shared = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThan, c, 1);
         self.b.ins().brif(shared, dec, &[], slow, &[]);
@@ -6624,6 +7984,141 @@ impl<'a> Lower<'a> {
         true
     }
 
+    /// A container instruction through one of the in-place helpers
+    /// ([`h_subscr_ref`], [`h_unpack_ref`], [`h_store_subscr_ref`]), its
+    /// `n` operands from `slot` up (the owned ones there, by the bits of
+    /// `owned`; `ops` their addresses and any further arguments): the
+    /// helper's new depth, a decline leaving for the core loop with the
+    /// operands on the stack, a raise (only with nothing virtual below
+    /// the operands) taking the handlers.
+    fn ref_call(
+        &mut self,
+        pc: usize,
+        helper: usize,
+        slot: usize,
+        n: usize,
+        ops: &[Value],
+        owned: u8,
+    ) {
+        let ins = self.b.ins().iconst(
+            self.ptr,
+            std::ptr::from_ref(&self.code.instructions[pc]) as i64,
+        );
+        let code = self.code_ptr();
+        let pcv = self.b.ins().iconst(types::I64, pc as i64);
+        let lenv = self.b.ins().iconst(types::I64, (slot + n) as i64);
+        let raisable = self.vs.iter().all(|i| matches!(i, Item::Mem(_)));
+        let flags = self
+            .b
+            .ins()
+            .iconst(types::I64, i64::from(owned) | (i64::from(raisable) << 8));
+        let mut args = vec![self.st, ins, code, pcv, lenv];
+        args.extend_from_slice(ops);
+        args.push(flags);
+        let r = self
+            .call_typed(helper, &args, Some(types::I64))
+            .expect("returns");
+        let declined = self.b.ins().icmp_imm(IntCC::Equal, r, -1);
+        let restore: Vec<Item> = (slot..slot + n).map(Item::Mem).collect();
+        let out = self.exit_with(pc, &restore, INTERP);
+        self.branch_out(declined, out);
+        if raisable {
+            let raised = self.b.ins().icmp_imm(IntCC::Equal, r, -2);
+            let raise_b = self.b.create_block();
+            let ok = self.b.create_block();
+            self.b.ins().brif(raised, raise_b, &[], ok, &[]);
+            self.b.switch_to_block(raise_b);
+            let s = self.b.ins().iconst(types::I32, i64::from(RAISED));
+            self.leave(s);
+            self.b.switch_to_block(ok);
+        }
+    }
+
+    /// `BINARY_SUBSCR` through [`h_subscr_ref`], which reads a local's or
+    /// constant's operand in place (no reference taken or released): the
+    /// item in the container's slot. `false`, emitting nothing, where the
+    /// depth after isn't known.
+    fn subscr_ref(&mut self, pc: usize) -> bool {
+        if self.depth < 2 || self.depths.get(pc + 1).copied().unwrap_or(-1) < 0 {
+            return false;
+        }
+        let key = self.pop();
+        let cont = self.pop();
+        let slot = self.depth;
+        let (pcont, own_c) = self.ref_or_own(cont, slot);
+        let (pkey, own_k) = self.ref_or_own(key, slot + 1);
+        let owned = u8::from(own_c) | (u8::from(own_k) << 1);
+        self.ref_call(
+            pc,
+            h_subscr_ref as *const () as usize,
+            slot,
+            2,
+            &[pcont, pkey],
+            owned,
+        );
+        self.push(Item::Mem(slot));
+        self.set_last(pc);
+        // (The helper released the owned operands; a finalizer that queued
+        // runs before the next instruction.)
+        self.check_released(pc + 1);
+        true
+    }
+
+    /// `UNPACK_SEQUENCE n` through [`h_unpack_ref`], which reads a local's
+    /// or constant's sequence in place. `false`, emitting nothing, where
+    /// the depth after isn't known.
+    fn unpack_ref(&mut self, pc: usize, n: usize) -> bool {
+        if n == 0 || self.depth == 0 || self.depths.get(pc + 1).copied().unwrap_or(-1) < 0 {
+            return false;
+        }
+        let item = self.pop();
+        let slot = self.depth;
+        let (src, owned) = self.ref_or_own(item, slot);
+        let nv = self.b.ins().iconst(types::I64, n as i64);
+        self.ref_call(
+            pc,
+            h_unpack_ref as *const () as usize,
+            slot,
+            1,
+            &[src, nv],
+            u8::from(owned),
+        );
+        for k in 0..n {
+            self.push(Item::Mem(slot + k));
+        }
+        self.set_last(pc);
+        self.check_released(pc + 1);
+        true
+    }
+
+    /// `STORE_SUBSCR` through [`h_store_subscr_ref`], which reads a local's
+    /// or constant's operand in place. `false`, emitting nothing, where
+    /// the depth after isn't known.
+    fn store_subscr_ref(&mut self, pc: usize) -> bool {
+        if self.depth < 3 || self.depths.get(pc + 1).copied().unwrap_or(-1) < 0 {
+            return false;
+        }
+        let key = self.pop();
+        let cont = self.pop();
+        let val = self.pop();
+        let slot = self.depth;
+        let (pv, own_v) = self.ref_or_own(val, slot);
+        let (pcont, own_c) = self.ref_or_own(cont, slot + 1);
+        let (pkey, own_k) = self.ref_or_own(key, slot + 2);
+        let owned = u8::from(own_v) | (u8::from(own_c) << 1) | (u8::from(own_k) << 2);
+        self.ref_call(
+            pc,
+            h_store_subscr_ref as *const () as usize,
+            slot,
+            3,
+            &[pv, pcont, pkey],
+            owned,
+        );
+        self.set_last(pc);
+        self.check_released(pc + 1);
+        true
+    }
+
     /// `LIST_APPEND` through [`h_list_append`] (the value leaves the stack
     /// into the list `arg` below it).
     fn list_append(&mut self, pc: usize, arg: usize) -> bool {
@@ -6640,16 +8135,40 @@ impl<'a> Lower<'a> {
         true
     }
 
-    /// `CONTAINS_OP` through [`h_contains`]: an unboxed `bool`.
+    /// An operand for a helper that reads it in place: a local's or a
+    /// constant's own value (borrowed, `false`), or anything else on the
+    /// stack at `slot` (owned there, `true`).
+    fn ref_or_own(&mut self, item: Item, slot: usize) -> (Value, bool) {
+        match item {
+            Item::Local(i) => (self.local_addr(i), false),
+            Item::Const(k) => (self.const_addr(k), false),
+            other => {
+                self.materialize(other, slot);
+                (self.slot_addr(slot), true)
+            }
+        }
+    }
+
+    /// `CONTAINS_OP` through [`h_contains_ref`], which reads a local or
+    /// constant operand in place (any other through [`h_contains`]): an
+    /// unboxed `bool`.
     fn contains(&mut self, pc: usize, arg: u32) -> bool {
         let bi = self.pop();
         let ai = self.pop();
         let slot = self.depth;
-        self.materialize(ai, slot);
-        self.materialize(bi, slot + 1);
+        let (pa, own_a) = self.ref_or_own(ai, slot);
+        let (pb, own_b) = self.ref_or_own(bi, slot + 1);
+        let owned = self.b.ins().iconst(
+            types::I32,
+            i64::from(u8::from(own_a) | (u8::from(own_b) << 1)),
+        );
         let at = self.slot_addr(slot);
         let r = self
-            .call(h_contains as *const () as usize, &[at], true)
+            .call(
+                h_contains_ref as *const () as usize,
+                &[pa, pb, owned, at],
+                true,
+            )
             .expect("returns");
         let declined = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThan, r, 1);
         let out = self.exit_with(pc, &[Item::Mem(slot), Item::Mem(slot + 1)], INTERP);
@@ -7527,10 +9046,16 @@ impl<'a> Lower<'a> {
         let pcv = self.b.ins().iconst(types::I64, pc as i64);
         let len = self.b.ins().iconst(types::I64, self.depth as i64);
         let c = self.b.ins().iconst(self.ptr, cache_at);
+        let mcache: Box<ModCache> = Box::default();
+        let mc = self
+            .b
+            .ins()
+            .iconst(self.ptr, std::ptr::from_ref::<ModCache>(&mcache) as i64);
+        self.mod_caches.push(mcache);
         let r = self
             .call(
                 h_load_method as *const () as usize,
-                &[self.st, code, ext, pcv, len, c],
+                &[self.st, code, ext, pcv, len, c, mc],
                 true,
             )
             .expect("returns");
@@ -7687,12 +9212,16 @@ impl<'a> Lower<'a> {
             .expect("returns");
         self.b.ins().jump(called, &[r.into()]);
         self.b.switch_to_block(general);
+        // (A site settled on a leaf builtin tries it first, in a smaller
+        // helper.)
+        let leaf = crate::call_site_leaf(self.ext, pc).is_some();
+        let helper = if leaf {
+            h_call_leaf as *const () as usize
+        } else {
+            h_call as *const () as usize
+        };
         let r = self
-            .call(
-                h_call as *const () as usize,
-                &[self.st, code, ext, pcv, len],
-                true,
-            )
+            .call(helper, &[self.st, code, ext, pcv, len], true)
             .expect("returns");
         self.b.ins().jump(called, &[r.into()]);
         self.b.switch_to_block(called);
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index b4904633..7fc6ae40 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -25165,6 +25165,7 @@ impl Interpreter {
                     ("any", LeafKind::Fast(crate::seqiter::any_fast)),
                     ("all", LeafKind::Fast(crate::seqiter::all_fast)),
                     ("abs", LeafKind::Fast(crate::seqiter::abs_fast)),
+                    ("hash", LeafKind::Fast(crate::seqiter::hash_fast)),
                     ("ord", LeafKind::Fast(crate::seqiter::ord_fast)),
                     ("chr", LeafKind::Fast(crate::seqiter::chr_fast)),
                     ("divmod", LeafKind::Fast(crate::seqiter::divmod_fast)),
@@ -25189,6 +25190,7 @@ impl Interpreter {
                         ("__len__", LeafKind::Len),
                         ("append", LeafKind::ListAppend),
                         ("pop", LeafKind::ListPop),
+                        ("extend", LeafKind::ListExtend),
                         ("insert", LeafKind::ListInsert),
                         ("reverse", LeafKind::ListReverse),
                         ("copy", LeafKind::ListCopy),
@@ -25882,6 +25884,13 @@ impl Interpreter {
                 });
             }
             K::ListInsert => args.len() == 3 && matches!(args[0], O::List(_)) && leaf_int(&args[1]),
+            // An exact list's or tuple's items, snapshotted: no iteration
+            // hook runs.
+            K::ListExtend => {
+                args.len() == 2
+                    && matches!(args[0], O::List(_))
+                    && matches!(args[1], O::List(_) | O::Tuple(_))
+            }
             K::ListReverse | K::ListCopy => args.len() == 1 && matches!(args[0], O::List(_)),
             // Admitted when every comparison `list.remove` makes up to its
             // match (or to the end: the ValueError) runs no Python code.
@@ -64337,6 +64346,8 @@ enum LeafKind {
     GetAttr,
     ListAppend,
     ListPop,
+    /// `list.extend` of an exact list or tuple.
+    ListExtend,
     ListInsert,
     ListReverse,
     ListCopy,
@@ -64389,6 +64400,7 @@ impl LeafKind {
                 | Self::SetAdd
                 | Self::ListAppend
                 | Self::ListPop
+                | Self::ListExtend
                 | Self::ListInsert
                 | Self::ListReverse
                 | Self::ListCopy
@@ -71437,6 +71449,51 @@ fn leaf_site_hit(
     }
 }
 
+/// The builtin method a `LOAD_ATTR` (method) site at `attr_pc` cached for
+/// a `list`, `dict`, `set` or `str` receiver, with the receiver's tag (see
+/// [`builtin_recv_tag`]) and the leaf kind its `CALL` at `call_pc`
+/// settled on, for the frame JIT's method kernels.
+#[cfg(feature = "jit")]
+pub(crate) fn builtin_method_site(
+    ext: &CodeConstObjects,
+    attr_pc: usize,
+    call_pc: usize,
+) -> Option<(u64, LeafKind, Rc<crate::object::BuiltinFn>)> {
+    let slots = ext.method_slots.get()?;
+    // SAFETY: GIL-serialized; the borrow ends before any refill.
+    let (ver, f) = unsafe { &*slots.get(attr_pc)?.0.get() };
+    let MethodSlotFn::Builtin(b) = f else {
+        return None;
+    };
+    if ver & MethodSlot::BUILTIN_TAG == 0 {
+        return None;
+    }
+    let tag = ver & !MethodSlot::BUILTIN_TAG;
+    if !(1..=4).contains(&tag) {
+        return None;
+    }
+    let kind = slots.get(call_pc)?.get_leaf_ptr(Rc::as_ptr(b))?;
+    Some((tag, kind, b.clone()))
+}
+
+/// The leaf builtin's kind the `CALL` at `pc` last settled on.
+#[cfg(feature = "jit")]
+pub(crate) fn call_site_leaf(ext: &CodeConstObjects, pc: usize) -> Option<LeafKind> {
+    let slot = ext.method_slots.get()?.get(pc)?;
+    // SAFETY: GIL-serialized; the borrow ends here.
+    match unsafe { &(*slot.0.get()).1 } {
+        MethodSlotFn::Leaf(_, kind) => Some(*kind),
+        _ => None,
+    }
+}
+
+/// Whether the `CALL` at `pc` last settled on a leaf builtin the core
+/// loop's arm (and the frame JIT's helper) runs in place.
+#[cfg(feature = "jit")]
+pub(crate) fn call_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
+    call_site_leaf(ext, pc).is_some_and(LeafKind::runs_in_core)
+}
+
 /// Whether the method call whose `LOAD_ATTR` is at `pc` last resolved to
 /// a callee the core loop runs in place (a builtin, or a leaf function),
 /// without an activation of its own.
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index c182d895..f15dda17 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -3357,7 +3357,7 @@ impl<'a> LeafProbe<'a> {
     pub fn new(key: &'a Object) -> Option<Self> {
         let hash = match key {
             Object::Str(s) => SharedStr::hash_cached(s),
-            Object::Int(_) => py_hash_value(key)?,
+            Object::Int(i) => py_hash_long_i64(*i),
             Object::Instance(i)
                 if i.native.get().is_none()
                     && i.class_dunder(crate::types::Dunder::Hash).object_owner() =>
@@ -4459,7 +4459,12 @@ impl Hash for DictKey {
         // (functions, types, plain instances, …) fold in their allocation
         // identity; truly unhashable keys share a constant bucket and the
         // runtime raises lazily when used.
-        let h = py_hash_value(&self.0).unwrap_or_else(|| identity_hash(&self.0));
+        let h = match &self.0 {
+            // (The commonest keys, without the general dispatch.)
+            Object::Str(s) => SharedStr::hash_cached(s),
+            Object::Int(i) => py_hash_long_i64(*i),
+            other => py_hash_value(other).unwrap_or_else(|| identity_hash(other)),
+        };
         h.hash(state);
     }
 }
@@ -12497,6 +12502,7 @@ pub(crate) fn py_hash_double(v: f64) -> i64 {
 
 /// CPython `long_hash` for a machine int: `sign * (|n| mod (2**61-1))`,
 /// with the reserved `-1` remapped to `-2`.
+#[inline]
 pub(crate) fn py_hash_long_i64(n: i64) -> i64 {
     // The modulus is the Mersenne prime `2**61 - 1`, so `|n| mod P` folds
     // the high bits onto the low ones instead of dividing: `|n| < 2**64`
@@ -13041,6 +13047,12 @@ fn frozenset_style_hash<I: Iterator<Item = i64>>(lanes: I) -> i64 {
 }
 
 pub(crate) fn py_hash_value(obj: &Object) -> Option<i64> {
+    // The commonest keys first, in line.
+    match obj {
+        Object::Int(i) => return Some(py_hash_long_i64(*i)),
+        Object::Str(s) => return Some(SharedStr::hash_cached(s)),
+        _ => {}
+    }
     if let Some(h) = numeric_hash(obj) {
         return Some(h);
     }
@@ -13081,6 +13093,24 @@ pub(crate) fn py_hash_value(obj: &Object) -> Option<i64> {
             if let Some(hash) = items.cached_hash() {
                 return Some(hash);
             }
+            // Ints and strings (the usual composite key): their hashes can
+            // neither fail nor run code, so no error scope is needed.
+            if items
+                .iter()
+                .all(|x| matches!(x, Object::Int(_) | Object::Str(_)))
+            {
+                let mut hash = TupleHasher::new();
+                for item in items.iter() {
+                    hash.push(match item {
+                        Object::Int(i) => py_hash_long_i64(*i),
+                        Object::Str(s) => SharedStr::hash_cached(s),
+                        _ => unreachable!("matched above"),
+                    });
+                }
+                let hash = hash.finish(items.len());
+                items.store_hash(hash);
+                return Some(hash);
+            }
             // Hash table callbacks communicate errors through the ambient
             // scope. Isolate this walk so a failed lane is never cached,
             // and stop before invoking any later element's hash method.
diff --git a/crates/weavepy-vm/src/seqiter.rs b/crates/weavepy-vm/src/seqiter.rs
index ad322e99..6b82238e 100644
--- a/crates/weavepy-vm/src/seqiter.rs
+++ b/crates/weavepy-vm/src/seqiter.rs
@@ -1591,6 +1591,27 @@ pub(crate) fn abs_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>>
     }))
 }
 
+/// `hash(x)` of a value whose hash runs no Python code and can't fail: a
+/// native scalar, a `str`, or a tuple of `int`s and `str`s.
+pub(crate) fn hash_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
+    let [x] = args else {
+        return None;
+    };
+    let native = match x {
+        Object::Int(_) | Object::Bool(_) | Object::None | Object::Str(_) => true,
+        // (A NaN hashes by identity.)
+        Object::Float(f) => !f.is_nan(),
+        Object::Tuple(t) => t
+            .iter()
+            .all(|e| matches!(e, Object::Int(_) | Object::Str(_))),
+        _ => false,
+    };
+    if !native {
+        return None;
+    }
+    Some(Ok(Object::Int(crate::object::py_hash_value(x)?)))
+}
+
 /// `ord(c)` of a one-character `str`.
 pub(crate) fn ord_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
     let [Object::Str(s)] = args else {
diff --git a/crates/weavepy-vm/src/shared_value.rs b/crates/weavepy-vm/src/shared_value.rs
index 007a4e43..63b19c42 100644
--- a/crates/weavepy-vm/src/shared_value.rs
+++ b/crates/weavepy-vm/src/shared_value.rs
@@ -831,6 +831,15 @@ impl SharedStr {
     }
 }
 
+impl SharedStr {
+    /// The shared one-character string of ASCII byte `c` (below `0x80`).
+    #[inline]
+    pub fn ascii_char(c: u8) -> Self {
+        debug_assert!(c.is_ascii());
+        small_table()[usize::from(c & 0x7f)].clone()
+    }
+}
+
 /// [`SharedStr::small`]'s strings: the 256 Latin-1 characters by code
 /// point, then the empty string.
 fn small_table() -> &'static [SharedStr; 257] {
diff --git a/tests/regrtest/test_frame_jit_containers.py b/tests/regrtest/test_frame_jit_containers.py
new file mode 100644
index 00000000..76e9c5eb
--- /dev/null
+++ b/tests/regrtest/test_frame_jit_containers.py
@@ -0,0 +1,228 @@
+"""The frame JIT's container shortcuts keep Python's semantics.
+
+Each function runs long enough to be compiled, then meets the shapes its
+in-place helpers and method kernels hand back to the general paths: keys
+that compare equal across types, subclass receivers with overrides,
+non-ASCII strings, out-of-range indices, wrong unpack lengths, misses that
+raise, and displaced values whose finalizers must run at once.
+"""
+
+import os
+import subprocess
+import sys
+import unittest
+
+N = 3000
+
+
+class Key:
+    """Equal to (and hashed like) the string it wraps."""
+
+    def __init__(self, s):
+        self.s = s
+
+    def __hash__(self):
+        return hash(self.s)
+
+    def __eq__(self, other):
+        return other == self.s or (isinstance(other, Key) and other.s == self.s)
+
+
+class Loud:
+    log = []
+
+    def __init__(self, name):
+        self.name = name
+
+    def __del__(self):
+        Loud.log.append(self.name)
+
+
+class MyList(list):
+    def append(self, x):
+        super().append(("wrapped", x))
+
+
+def method_calls(n, xs, d, s, text):
+    out = []
+    for i in range(n):
+        xs.append(i)
+        xs.pop()
+        out.append(d.get("a"))
+        out.append(d.get("zz", -1))
+        s.add(i & 7)
+        s.discard(i & 7)
+        out.append(text.startswith("ab"))
+        out.append(text.endswith("yz"))
+        out.append(text.isdigit())
+        out.append(text.find("c"))
+        out.append(text.strip())
+        out.append(text.upper())
+    return out
+
+
+def subscripts(n, d, xs, t, text, key):
+    total = 0
+    for i in range(n):
+        total += d[key] + xs[1] + xs[-1] + t[0]
+        total += len(text[2])
+    return total
+
+
+def contains(n, item, seq):
+    hits = 0
+    for i in range(n):
+        if item in seq:
+            hits += 1
+    return hits
+
+
+def unpacks(n, seq):
+    total = 0
+    for i in range(n):
+        a, b, c = seq
+        total += a + b + c
+    return total
+
+
+def stores(n, d, xs, key, value):
+    for i in range(n):
+        d[key] = value
+        xs[0] = value
+    return d, xs
+
+
+def iterate(n, text):
+    out = []
+    for i in range(n):
+        for ch in text:
+            out.append(ch)
+    return out
+
+
+def hashes(n, value):
+    h = 0
+    for i in range(n):
+        h = hash(value)
+    return h
+
+
+class FrameJitContainerTests(unittest.TestCase):
+    def test_method_kernels(self):
+        out = method_calls(N, [1, 2], {"a": 1}, set(), "abcxyz")
+        self.assertEqual(out[:8], [1, -1, True, True, False, 2, "abcxyz", "ABCXYZ"])
+        # Shapes the kernels decline still run correctly.
+        out = method_calls(5, MyList(), {1.0: "f", "a": 2}, {True}, " ¹²³ ")
+        self.assertEqual(out[:8], [2, -1, False, False, False, -1, "¹²³", " ¹²³ "])
+        self.assertTrue("٣٤".isdigit())
+        self.assertTrue("\x1c\x1f ".isspace())
+        self.assertEqual("héllo".find("l"), 2)
+
+    def test_list_kernels(self):
+        xs = [1, 2]
+        for _ in range(N):
+            xs.extend(())
+        xs.extend(xs)
+        self.assertEqual(xs, [1, 2, 1, 2])
+        m = MyList()
+        for i in range(3):
+            m.append(i)
+        self.assertEqual(m, [("wrapped", 0), ("wrapped", 1), ("wrapped", 2)])
+        empty = []
+        with self.assertRaises(IndexError):
+            for _ in range(N):
+                empty.pop()
+
+    def test_set_kernels(self):
+        s = {True}
+        for _ in range(N):
+            s.add(1)
+        self.assertEqual(s, {True})
+        self.assertIs(next(iter(s)), True)
+        s.discard(1)
+        self.assertEqual(s, set())
+        with self.assertRaises(KeyError):
+            for i in range(N):
+                s.add(i)
+                s.remove(i)
+            s.remove(-5)
+        t = {Key("a")}
+        for _ in range(N):
+            t.add("b")
+            t.discard("b")
+        t.discard("a")
+        self.assertEqual(t, set())
+
+    def test_dict_get_kernel(self):
+        d = {Key("k"): 5, 2.0: "two"}
+        for _ in range(N):
+            v = d.get("k")
+        self.assertEqual(v, 5)
+        self.assertEqual(d.get(2), "two")
+
+    def test_subscripts(self):
+        self.assertEqual(subscripts(4 * N, {"k": 1}, [1, 2, 3], (4,), "abcd", "k"), 4 * N * 11)
+        with self.assertRaises(KeyError):
+            subscripts(1, {}, [1, 2, 3], (4,), "abcd", "k")
+        with self.assertRaises(IndexError):
+            subscripts(1, {"k": 1}, [1], (4,), "abcd", "k")
+        self.assertEqual(subscripts(2, {Key("k"): 1}, [1, 2, 3], (4,), "héllo", "k"), 22)
+        self.assertEqual(subscripts(2, {(1, "a"): 1}, [1, 2, 3], (4,), "abcd", (1, "a")), 22)
+
+    def test_contains(self):
+        self.assertEqual(contains(N, "c", "abcd"), N)
+        self.assertEqual(contains(N, 3, {1.5, 3.0}), N)
+        self.assertEqual(contains(N, "a", {Key("a")}), N)
+        self.assertEqual(contains(10, Key("x"), ["y", "x"]), 10)
+        with self.assertRaises(TypeError):
+            contains(1, 3, "abc")
+
+    def test_unpacks(self):
+        self.assertEqual(unpacks(N, (1, 2, 3)), N * 6)
+        self.assertEqual(unpacks(N, [1, 2, 3]), N * 6)
+        self.assertEqual(unpacks(3, range(1, 4)), 18)
+        with self.assertRaises(ValueError):
+            unpacks(1, (1, 2))
+        with self.assertRaises(ValueError):
+            unpacks(1, [1, 2, 3, 4])
+
+    def test_stores(self):
+        d, xs = stores(4 * N, {}, [0], "k", 7)
+        self.assertEqual((d, xs), ({"k": 7}, [7]))
+        d, xs = stores(1, {Key("k"): 0}, [0], "k", 1)
+        self.assertEqual(list(d.values()), [1])
+        with self.assertRaises(IndexError):
+            stores(1, {}, [], "k", 1)
+        # A displaced value's finalizer runs before the next statement.
+        Loud.log.clear()
+        d = {"k": Loud("old")}
+        xs = [Loud("old-item")]
+        stores(1, d, xs, "k", 0)
+        self.assertEqual(Loud.log, ["old", "old-item"])
+
+    def test_iteration(self):
+        self.assertEqual(iterate(N, "ab")[:4], ["a", "b", "a", "b"])
+        self.assertEqual(iterate(1, "héé"), ["h", "é", "é"])
+
+    def test_hash(self):
+        self.assertEqual(hashes(N, -1), -2)
+        self.assertEqual(hashes(N, (1, "a")), hash((1, "a")))
+        self.assertEqual({(1, "a"): 1}[(1, "a")], 1)
+        nan = float("nan")
+        self.assertEqual(hashes(3, nan), hash(nan))
+        self.assertEqual(hashes(3, (1, 2.5)), hash((1, 2.5)))
+
+
+    @unittest.skipIf(os.environ.get("WEAVEPY_TIER2") == "0", "already without tier 2")
+    def test_without_tier2(self):
+        # Tier 2 takes some of these loops first: run them all again with
+        # it off, so the frame JIT compiles every one.
+        env = dict(os.environ, WEAVEPY_TIER2="0")
+        r = subprocess.run(
+            [sys.executable, __file__], env=env, capture_output=True, text=True, timeout=120
+        )
+        self.assertEqual(r.returncode, 0, r.stderr[-2000:])
+
+
+if __name__ == "__main__":
+    unittest.main()

From f8407c9abc9911f4d11769c20b0e381a9e3cb6a5 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 20:24:13 -0700
Subject: [PATCH 015/173] perf: take a local container's other subscripts in
 place too

The in-line list/tuple index of a local container sent every other shape
(a dict, a string, an index out of range) to the general container
helper with both operands copied to the stack and released after. It now
takes the in-place subscript helper, which reads the local where it lies
and settles exact dicts and strings itself, falling back to the
container helper inside.

[s for s in u if '5' in d[s]] over 27 keys: 27371 -> 22614 instructions
(CPython 14303); d[key] on an 81-key dict 437 -> 414; tomllib_loads
0.959x, the suite geomean unchanged (0.9998).
---
 crates/weavepy-vm/src/frame_jit.rs | 53 +++++++++---------------------
 1 file changed, 15 insertions(+), 38 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index f2727c50..4a954b8c 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -7842,8 +7842,8 @@ impl<'a> Lower<'a> {
     /// `local[i]` of a list or tuple local and an `int` index, in line:
     /// the item is read in place (the local gives no reference to take or
     /// release) and copied to the stack. Any other operands (a `str` or a
-    /// dict, a borrowed list, an index out of range) take the container
-    /// helper, as [`Self::container`] would. `false`, emitting nothing, for
+    /// dict, a borrowed list, an index out of range) take the in-place
+    /// helper, as [`Self::subscr_ref`] does. `false`, emitting nothing, for
     /// other operand shapes.
     fn seq_index(&mut self, pc: usize) -> bool {
         let (Some(list), Some((tlen, titems))) = (self.tags.list_layout, self.tags.tuple_layout)
@@ -7933,44 +7933,21 @@ impl<'a> Lower<'a> {
         let dst = self.slot_addr(slot);
         self.copy_value(dst, item);
         self.b.ins().jump(join, &[]);
-        // Anything else: both operands to the stack, and the helper.
+        // Anything else (a dict, a string, a borrowed list, an index out of
+        // range) through the in-place helper, which reads the local
+        // container where it lies (see `Self::subscr_ref`).
         self.b.switch_to_block(slow);
-        self.materialize(cont, slot);
-        self.materialize(key, slot + 1);
-        let ins = self.b.ins().iconst(
-            self.ptr,
-            std::ptr::from_ref(&self.code.instructions[pc]) as i64,
+        let (pcont, own_c) = self.ref_or_own(cont, slot);
+        let (pkey, own_k) = self.ref_or_own(key, slot + 1);
+        let owned = u8::from(own_c) | (u8::from(own_k) << 1);
+        self.ref_call(
+            pc,
+            h_subscr_ref as *const () as usize,
+            slot,
+            2,
+            &[pcont, pkey],
+            owned,
         );
-        let code = self
-            .b
-            .ins()
-            .iconst(self.ptr, std::ptr::from_ref(self.code) as i64);
-        let pcv = self.b.ins().iconst(types::I64, pc as i64);
-        let lenv = self.b.ins().iconst(types::I64, slot as i64 + 2);
-        // (A raise needs the whole stack on the frame's: nothing virtual
-        // below the operands.)
-        let raisable = self.vs.iter().all(|i| matches!(i, Item::Mem(_)));
-        let can_raise = self.b.ins().iconst(types::I64, i64::from(raisable));
-        let r = self
-            .call_typed(
-                h_container as *const () as usize,
-                &[self.st, ins, code, pcv, lenv, can_raise],
-                Some(types::I64),
-            )
-            .expect("returns");
-        let declined = self.b.ins().icmp_imm(IntCC::Equal, r, -1);
-        let out = self.exit_with(pc, &[Item::Mem(slot), Item::Mem(slot + 1)], INTERP);
-        self.branch_out(declined, out);
-        if raisable {
-            let raised = self.b.ins().icmp_imm(IntCC::Equal, r, -2);
-            let raise_b = self.b.create_block();
-            let ok = self.b.create_block();
-            self.b.ins().brif(raised, raise_b, &[], ok, &[]);
-            self.b.switch_to_block(raise_b);
-            let s = self.b.ins().iconst(types::I32, i64::from(RAISED));
-            self.leave(s);
-            self.b.switch_to_block(ok);
-        }
         // (The helper released the operands; a finalizer it queued runs
         // before the next instruction, as after `Self::container`.)
         self.depth = slot + 1;

From 1f49693345697c4e82c60eeaf4a21b30635436d1 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 20:41:47 -0700
Subject: [PATCH 016/173] perf: run settled leaf bodies from the leaf-site call
 helper

A call site settled on a registered leaf body (a deque's append or
popleft, a pure str method on a stack receiver) or a scalar-only builtin
went from the leaf-site helper on to the general call helper, which
looked the site up again before running the same body. The leaf-site
helper now runs those bodies itself, under the general helper's checks.

deque append+popleft (microops, globals): 1852 -> 1473 instructions;
math.floor(2.5) 764 -> 551. Suite geomean 0.9963 against the previous
commit (0.9715 against the branch's baseline).
---
 crates/weavepy-vm/src/frame_jit.rs | 36 +++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 4a954b8c..5caa061b 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -4556,10 +4556,11 @@ fn len_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
 }
 
 /// [`h_call`] at a site settled on a leaf builtin: a container's or
-/// string's method through its [`Kernel`], or `isinstance`, `len` or a
-/// registered fast half in place as `h_call` runs them first, from a
-/// small function (the general helper stays as it is); anything else
-/// (another callee, a declined shape) through `h_call`.
+/// string's method through its [`Kernel`], or `isinstance`, `len`, a
+/// registered fast half or a registered leaf body in place as `h_call`
+/// runs them, from a small function (the general helper stays as it
+/// is); anything else (another callee, a declined shape) through
+/// `h_call`.
 unsafe extern "C" fn h_call_leaf(
     st: *mut State,
     code: *const CodeObject,
@@ -4599,7 +4600,13 @@ unsafe extern "C" fn h_call_leaf(
                 }
                 if matches!(
                     kind,
-                    Some(LeafKind::Isinstance | LeafKind::Len | LeafKind::Fast(_))
+                    Some(
+                        LeafKind::Isinstance
+                            | LeafKind::Len
+                            | LeafKind::Fast(_)
+                            | LeafKind::Opaque
+                            | LeafKind::Scalar
+                    )
                 ) && Rc::strong_count(b) > 1
                 {
                     let first = if matches!(&*base.add(start + 1), Object::Unbound) {
@@ -4609,9 +4616,28 @@ unsafe extern "C" fn h_call_leaf(
                     };
                     let args = std::slice::from_raw_parts(base.add(first), lenu - first);
                     let r = if Interpreter::core_all_droppable(args) {
+                        // (A body that runs no Python code for any
+                        // arguments, or for scalar ones: as `h_call` and
+                        // `leaf_builtin_call` run them.)
+                        let body = || {
+                            Some(match b.call_kw.as_ref() {
+                                Some(ckw) => ckw(args, &[]),
+                                None => (b.call)(args),
+                            })
+                        };
                         match kind {
                             Some(LeafKind::Fast(f)) => f(args),
                             Some(LeafKind::Len) => len_fast(args),
+                            Some(LeafKind::Opaque) => body(),
+                            Some(LeafKind::Scalar) => {
+                                if args.iter().all(|a| {
+                                    matches!(a, Object::Int(_) | Object::Float(_) | Object::Bool(_))
+                                }) {
+                                    body()
+                                } else {
+                                    None
+                                }
+                            }
                             _ => Interpreter::core_isinstance(args),
                         }
                     } else {

From bdde30dcfb2f8217d8e962e71ad2b772f6c671ee Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 22:10:12 -0700
Subject: [PATCH 017/173] perf: call effect leaves that call directly

A method call of an effect leaf (a body that stores) whose body itself
makes a call no longer runs frameless through the leaf evaluator: its
callee's evaluation nests inside it, and the whole costs more than the
in-line method load and direct call compiled code otherwise takes. Pure
leaves, and effect leaves that make no call (setters, counters), keep
the frameless path. Each code object caches whether it calls.

Instructions retired: go -6.9% (its recursive Square.find), raytrace
-1.0%; the rest within noise.
---
 crates/weavepy-vm/src/lib.rs | 45 +++++++++++++++++++++++++++++++-----
 1 file changed, 39 insertions(+), 6 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 7fc6ae40..3c9a4578 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -11080,11 +11080,16 @@ impl Interpreter {
             .map_or(&[][..], |s| &s[..]);
         match other {
             Object::Instance(_) | Object::Module(_) => {
-                // A site that verified its callee for this class version
-                // (its call checks the argument shape itself).
+                // A site that verified its pure callee for this class version
+                // (its call checks the argument shape itself). An effect
+                // leaf (one that stores) costs more evaluated frameless than
+                // called: the method loads and its call runs directly.
                 let mut missed = false;
                 if let (Object::Instance(i), Some(ext)) = (other, ext) {
-                    if let Some(site) = leaf_site_hit(ext, pc + 1, i.cls_raw().attr_version.get()) {
+                    if let Some(site) = leaf_site_hit(ext, pc + 1, i.cls_raw().attr_version.get())
+                        // SAFETY: the site's code is its live function's.
+                        .filter(|&(_, c, effect)| !effect || !code_calls(unsafe { &*c }))
+                    {
                         match self.core_leaf_site_call(
                             code,
                             i,
@@ -11136,7 +11141,11 @@ impl Interpreter {
                             .peek_fn(i.cls_raw().attr_version.get())
                             // SAFETY: the slot's function is alive (its class
                             // version matched).
-                            .is_some_and(|fp| fn_is_leaf(unsafe { &*fp })),
+                            .is_some_and(|fp| {
+                                let f = unsafe { &*fp };
+                                fn_is_pure_leaf(f)
+                                    || (fn_is_leaf(f) && !code_calls(unsafe { &*f.code.as_ptr() }))
+                            }),
                         _ => false,
                     };
                 if pure_site {
@@ -56330,8 +56339,7 @@ impl Interpreter {
         // SAFETY: GIL-serialized raw read of the function's code cell (the
         // callee on the stack holds the function).
         let code_ref: &Rc<CodeObject> = unsafe { &*f.code.as_ptr() };
-        let Some(start) =
-            self.generator_start(f, code_ref, argc + usize::from(receiver.is_some()))
+        let Some(start) = self.generator_start(f, code_ref, argc + usize::from(receiver.is_some()))
         else {
             return false;
         };
@@ -70938,6 +70946,9 @@ struct CodeConstObjects {
     /// The body's store-only `__init__` shape (see [`code_store_init`]),
     /// or `None` when it isn't one.
     store_init: std::sync::OnceLock<Option<Box<StoreInit>>>,
+    /// Whether the body makes a call (see [`code_calls`]): `0` not yet
+    /// decided, `1` no, `2` yes.
+    calls: std::sync::atomic::AtomicU8,
     /// Per `co_names` entry, its position among the shared names of the
     /// instances last checked for shadowing it (see [`inst_may_shadow`]).
     name_memos: std::sync::OnceLock<Box<[crate::inst_dict::NameMemo]>>,
@@ -72786,6 +72797,27 @@ fn code_returns_only_none(code: &CodeObject) -> bool {
     yes
 }
 
+/// Whether `code`'s body makes a call of any kind (cached in its
+/// extension). An effect leaf that calls costs more evaluated frameless,
+/// its callee's evaluation nested in it, than called directly.
+fn code_calls(code: &CodeObject) -> bool {
+    use std::sync::atomic::Ordering::Relaxed;
+    let Some(ext) = code_vm_ext(code) else {
+        return true;
+    };
+    match ext.calls.load(Relaxed) {
+        1 => return false,
+        2 => return true,
+        _ => {}
+    }
+    let yes = code
+        .instructions
+        .iter()
+        .any(|i| matches!(i.op, OpCode::Call | OpCode::CallKw | OpCode::CallEx));
+    ext.calls.store(if yes { 2 } else { 1 }, Relaxed);
+    yes
+}
+
 /// Where a store-only `__init__`'s stored value comes from.
 #[derive(Clone, Copy)]
 enum StoreSrc {
@@ -73084,6 +73116,7 @@ fn code_vm_ext_build(code: &CodeObject) -> &CodeConstObjects {
             leaf_plan: std::sync::OnceLock::new(),
             returns_none: std::sync::atomic::AtomicU8::new(0),
             store_init: std::sync::OnceLock::new(),
+            calls: std::sync::atomic::AtomicU8::new(0),
             name_memos: std::sync::OnceLock::new(),
             gen_names: std::sync::OnceLock::new(),
             #[cfg(feature = "jit")]

From 1e3849b4fc1b07994f9c55b7e5ab091f44a74551 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 22:50:58 -0700
Subject: [PATCH 018/173] perf: bind a direct call's locals in one pass

The plain call binder moved the arguments, then pushed the missing
defaults, then filled the other locals with unbound markers, each with
its own capacity check and loop. It now checks the capacity once and
writes all three in one pass.
---
 crates/weavepy-vm/src/lib.rs | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 3c9a4578..6f27556b 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -15455,28 +15455,38 @@ impl Interpreter {
             callee_slot + 2
         };
         let nargs = n - first;
-        locals.reserve(nlocals.max(nargs + missing));
+        let total = nlocals.max(nargs + missing);
+        if locals.capacity() < total {
+            locals.reserve(total);
+        }
         // SAFETY: the `nargs` operands above `first` move into the locals
-        // (reserved above) and the stack forgets them; the callable moves
-        // out of its slot, and an empty self slot owns nothing.
+        // (reserved above) and the stack forgets them, the defaults follow
+        // as copies, and the other locals start unbound, in one pass; the
+        // callable moves out of its slot, and an empty self slot owns
+        // nothing.
         let callable = unsafe {
             let (src, dst) = (frame.stack.as_ptr().add(first), locals.as_mut_ptr());
-            for k in 0..nargs {
+            let mut k = 0;
+            while k < nargs {
                 dst.add(k).write(src.add(k).read());
+                k += 1;
             }
-            locals.set_len(nargs);
+            if missing > 0 {
+                let defaults = &(*fp).defaults;
+                for d in &defaults[defaults.len() - missing..] {
+                    dst.add(k).write(clone_hot(d));
+                    k += 1;
+                }
+            }
+            while k < total {
+                dst.add(k).write(Object::Unbound);
+                k += 1;
+            }
+            locals.set_len(total);
             let callable = frame.stack.as_ptr().add(callee_slot).read();
             frame.stack.set_len(callee_slot);
             callable
         };
-        if missing > 0 {
-            // SAFETY: as above.
-            let defaults = unsafe { &(*fp).defaults };
-            for d in &defaults[defaults.len() - missing..] {
-                push_fast(locals, clone_hot(d));
-            }
-        }
-        fill_unbound(locals, nlocals);
         frame.pc = pc as u32 + 1;
         // SAFETY: the slot is parked (see `inline_bind_cells`): its handles
         // are stale copies, overwritten without a drop. `fp` stays alive

From 6b76cbd0762ef45c28ba99c07bfae68a6c3ddcc5 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 23:31:24 -0700
Subject: [PATCH 019/173] perf: keep two callees per direct-call site

A CALL site's direct-call cache held one callee, so a polymorphic method
call (an expression tree's eval on its node classes) refilled it on
every change of receiver, and a callee without a compiled body (a
frameless leaf) emptied it, sending the compiled callees through the
core loop for the next 64 calls. The site now keeps a second callee,
promotes a newly seen compiled one to first place, and leaves its
compiled callees alone when one without a body comes by.

mini_interpreter's direct calls rise from 6.5k to 35k per run.
---
 crates/weavepy-vm/src/frame_jit.rs | 55 ++++++++++++++++++++++++------
 1 file changed, 45 insertions(+), 10 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 5caa061b..0cd7da47 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3732,6 +3732,13 @@ pub(crate) struct DirectSite {
     /// How many trailing parameters take the function's defaults.
     missing: u32,
     has_self: bool,
+    /// A second callee the site calls directly (a polymorphic method
+    /// call's other receiver class): its code (a counted reference, `0`
+    /// none), body, defaults taken and self slot.
+    alt_code: usize,
+    alt_native: usize,
+    alt_missing: u32,
+    alt_has_self: bool,
 }
 
 const DS_NATIVE: i32 = std::mem::offset_of!(DirectSite, native) as i32;
@@ -3739,9 +3746,11 @@ const DS_RETRY: i32 = std::mem::offset_of!(DirectSite, retry) as i32;
 
 impl Drop for DirectSite {
     fn drop(&mut self) {
-        if self.code != 0 {
-            // SAFETY: `code` holds the count `fill` took.
-            unsafe { drop(Rc::from_raw(self.code as *const CodeObject)) };
+        for code in [self.code, self.alt_code] {
+            if code != 0 {
+                // SAFETY: each holds the count a fill took.
+                unsafe { drop(Rc::from_raw(code as *const CodeObject)) };
+            }
         }
     }
 }
@@ -3762,6 +3771,24 @@ impl DirectSite {
         self.native = native;
         self.retry = if native == 0 { 64 } else { 0 };
     }
+
+    /// Record a call of `code` with a native body as the site's first
+    /// callee, the one it held before (if it had a body) becoming its
+    /// second.
+    fn promote(&mut self, code: &Rc<CodeObject>, has_self: bool, native: usize, missing: usize) {
+        if self.native != 0 && self.code != 0 && self.code != Rc::as_ptr(code) as usize {
+            let old = std::mem::replace(&mut self.alt_code, self.code);
+            self.code = 0;
+            if old != 0 {
+                // SAFETY: `old` held the count an earlier fill took.
+                unsafe { drop(Rc::from_raw(old as *const CodeObject)) };
+            }
+            self.alt_native = self.native;
+            self.alt_missing = self.missing;
+            self.alt_has_self = self.has_self;
+        }
+        self.fill(code, has_self, native, missing);
+    }
 }
 
 /// Whether a call of `f` (running `code`) with `nargs` positional
@@ -4039,26 +4066,34 @@ unsafe fn direct_call(
         // (The site keys on the code: a fresh function made from the same
         // `def` binds the same way, and code without free variables takes
         // no closure.)
-        if site.code != Rc::as_ptr(code_rc) as usize
-            || site.has_self != has_self
-            || site.native == 0
+        let at = Rc::as_ptr(code_rc) as usize;
+        let (native, missing) = if site.code == at && site.has_self == has_self && site.native != 0
         {
+            (site.native, site.missing as usize)
+        } else if site.alt_code == at && site.alt_has_self == has_self && site.alt_native != 0 {
+            (site.alt_native, site.alt_missing as usize)
+        } else {
             let nargs = len - start - 2 + usize::from(has_self);
             let (native, missing) = direct_body(f, code_rc, nargs).map_or((0, 0), |(n, m)| {
                 (std::ptr::from_ref::<Native>(n) as usize, m)
             });
-            site.fill(code_rc, has_self, native, missing);
             if native == 0 {
+                // A callee with no body (a frameless leaf) leaves the site's
+                // compiled callees alone.
+                if site.native == 0 {
+                    site.fill(code_rc, has_self, 0, 0);
+                }
                 return None;
             }
-        }
+            site.promote(code_rc, has_self, native, missing);
+            (native, missing)
+        };
         // (Another function running the same code may have rebound its
         // defaults.)
-        let missing = site.missing as usize;
         if missing > 0 && !Interpreter::defaults_cover(f, missing) {
             return None;
         }
-        let native = &*(site.native as *const Native);
+        let native = &*(native as *const Native);
         let interp = &mut *st.interp.cast_mut();
         let nested = interp.direct_calls.get();
         let code: &CodeObject = code_rc;

From 0eb82d24d5bb0c1201d74d09dc06275d0e3d428f Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 23:59:42 -0700
Subject: [PATCH 020/173] perf: keep a method site's other receiver classes

A method load site's in-line cache holds one receiver class, so a
polymorphic site (an expression tree's eval, shapes' area) took the
general helper whenever the class changed. The cache now keeps up to
three more classes' functions, each with the shared-names position
that proves an instance doesn't shadow the method, and the helper
checks them before its general path.

Instructions retired: mini_interpreter -2.9%, oo_patterns -0.5%.
---
 crates/weavepy-vm/src/frame_jit.rs            | 66 +++++++++++++++++--
 crates/weavepy-vm/src/inst_dict.rs            | 13 ++++
 .../regrtest/test_polymorphic_method_loads.py | 66 +++++++++++++++++++
 3 files changed, 138 insertions(+), 7 deletions(-)
 create mode 100644 tests/regrtest/test_polymorphic_method_loads.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 0cd7da47..22750282 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3135,6 +3135,34 @@ unsafe extern "C" fn h_load_method(
             st.stack.add(len).write(Object::Unbound);
             return 0;
         }
+        // A receiver class the site saw before: its cached function, while
+        // the instance holds no dictionary and no value at or past the
+        // name's position (see `Lower::load_method`).
+        if let (Object::Instance(inst), false) = (&*top, cache.is_null()) {
+            let ver = inst.cls_raw().attr_version.get();
+            if let Some(e) = (*cache).poly.iter().find(|e| e.ver == ver) {
+                let shadow_free = inst.dict.published().is_none()
+                    && match inst.dict.split_peek() {
+                        Some(split) => {
+                            let (keys, n) = split.keys_and_len();
+                            n == 0 || (keys as u64 == e.keys && n <= e.before as usize)
+                        }
+                        None => false,
+                    };
+                if shadow_free {
+                    // The function's own bits: its class holds it while the
+                    // version stands, so a clone takes a count of it.
+                    let f = std::mem::ManuallyDrop::new(
+                        std::ptr::from_ref(&e.obj).cast::<Object>().read(),
+                    );
+                    let f = crate::clone_hot(&f);
+                    let recv = top.read();
+                    top.write(f);
+                    st.stack.add(len).write(recv);
+                    return 0;
+                }
+            }
+        }
         let Some(ms) = ext.method_slots.get().and_then(|s| s.get(pc)) else {
             return 1;
         };
@@ -3219,6 +3247,21 @@ pub(crate) struct MethodCache {
     func: u64,
     keys: u64,
     before: u32,
+    /// The function `Object`'s first word (its tag), `func` its second.
+    head: u64,
+    /// The site's other receiver classes (a polymorphic call), checked by
+    /// the helper before its general path, most recent first.
+    poly: [MethodEntry; 3],
+}
+
+/// One of a [`MethodCache`]'s polymorphic entries.
+#[derive(Default, Clone, Copy)]
+struct MethodEntry {
+    ver: u64,
+    /// The function `Object`'s two words.
+    obj: [u64; 2],
+    keys: u64,
+    before: u32,
 }
 
 const MC_VER: i32 = std::mem::offset_of!(MethodCache, ver) as i32;
@@ -3255,14 +3298,23 @@ fn fill_method_cache(
     };
     let obj = Object::Function(f.clone());
     // SAFETY: an `Object` is 16 bytes; its second word is the payload.
-    let word = unsafe { std::ptr::from_ref(&obj).cast::<u64>().add(1).read() };
+    let [head, word] = unsafe { std::ptr::from_ref(&obj).cast::<[u64; 2]>().read() };
     drop(obj);
-    *cache = MethodCache {
-        ver,
-        func: word,
-        keys,
-        before,
-    };
+    // The entry it replaces moves down the polymorphic list.
+    if cache.ver != 0 && cache.ver != ver {
+        cache.poly.copy_within(0..2, 1);
+        cache.poly[0] = MethodEntry {
+            ver: cache.ver,
+            obj: [cache.head, cache.func],
+            keys: cache.keys,
+            before: cache.before,
+        };
+    }
+    cache.ver = ver;
+    cache.head = head;
+    cache.func = word;
+    cache.keys = keys;
+    cache.before = before;
 }
 
 /// A method load's module-attribute shortcut: the module dict's address
diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index 9bf1f2e4..457fd71e 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -314,6 +314,19 @@ impl SplitValues {
         unsafe { (*h).keys.as_ref() }
     }
 
+    /// The shared names' address (`0` none) and how many values are set.
+    #[inline(always)]
+    pub fn keys_and_len(&self) -> (usize, usize) {
+        match self.block {
+            // SAFETY: the block is live while owned.
+            Some(b) => unsafe {
+                let h = b.as_ptr();
+                ((*h).keys as usize, (*h).len as usize)
+            },
+            None => (0, 0),
+        }
+    }
+
     /// How many attributes are set.
     #[inline(always)]
     pub fn len(&self) -> usize {
diff --git a/tests/regrtest/test_polymorphic_method_loads.py b/tests/regrtest/test_polymorphic_method_loads.py
new file mode 100644
index 00000000..c6a12da0
--- /dev/null
+++ b/tests/regrtest/test_polymorphic_method_loads.py
@@ -0,0 +1,66 @@
+# A method load site that sees several receiver classes keeps each one's
+# method, and must still notice an instance attribute that shadows the
+# method, a method replaced on its class, and a class reassigned on an
+# instance.
+
+
+class A:
+    def __init__(self):
+        self.v = 1
+
+    def get(self):
+        return "A"
+
+
+class B:
+    def __init__(self):
+        self.v = 2
+
+    def get(self):
+        return "B"
+
+
+class C(B):
+    def get(self):
+        return "C"
+
+
+class D:
+    def get(self):
+        return "D"
+
+
+def run(objs, n=200):
+    out = []
+    for _ in range(n):
+        out = [o.get() for o in objs]
+    return out
+
+
+objs = [A(), B(), C(), D()]
+assert run(objs) == ["A", "B", "C", "D"]
+
+# An instance attribute shadows its class's method.
+b2 = B()
+b2.get = lambda: "shadow"
+assert run([A(), b2, C(), D()]) == ["A", "shadow", "C", "D"]
+
+# A method replaced on its class (and on a base class).
+B.get = lambda self: "B2"
+assert run(objs) == ["A", "B2", "C", "D"]
+del C.get
+assert run(objs) == ["A", "B2", "B2", "D"]
+
+# A class reassigned on an instance.
+a = A()
+a.__class__ = D
+assert run([a, objs[1], objs[2], objs[3]]) == ["D", "B2", "B2", "D"]
+
+# An attribute added after the site cached the method, on an instance with
+# split values of its own.
+late = A()
+assert run([late, objs[1]]) == ["A", "B2"]
+late.get = lambda: "late"
+assert run([late, objs[1]]) == ["late", "B2"]
+
+print("test_polymorphic_method_loads: OK")

From 29231dd02989f1e43eec8717017645cc073c45b1 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 00:46:07 -0700
Subject: [PATCH 021/173] perf: move a call's few arguments in straight-line
 code

LLVM unrolled the binder's argument move and unbound fill by four, so a
call with one or two arguments paid each loop's prologue and remainder
(about 55 instructions a call between them). Counts up to three move in
straight-line code now.
---
 crates/weavepy-vm/src/lib.rs | 66 +++++++++++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 9 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 6f27556b..2621f98f 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -15466,11 +15466,8 @@ impl Interpreter {
         // nothing.
         let callable = unsafe {
             let (src, dst) = (frame.stack.as_ptr().add(first), locals.as_mut_ptr());
-            let mut k = 0;
-            while k < nargs {
-                dst.add(k).write(src.add(k).read());
-                k += 1;
-            }
+            move_objects(src, dst, nargs);
+            let mut k = nargs;
             if missing > 0 {
                 let defaults = &(*fp).defaults;
                 for d in &defaults[defaults.len() - missing..] {
@@ -15478,10 +15475,7 @@ impl Interpreter {
                     k += 1;
                 }
             }
-            while k < total {
-                dst.add(k).write(Object::Unbound);
-                k += 1;
-            }
+            fill_unbound_raw(dst.add(k), total - k);
             locals.set_len(total);
             let callable = frame.stack.as_ptr().add(callee_slot).read();
             frame.stack.set_len(callee_slot);
@@ -65981,6 +65975,60 @@ fn is_object_new(b: &Rc<crate::object::BuiltinFn>) -> bool {
     Rc::as_ptr(b) as usize == want
 }
 
+/// Move `n` objects from `src` to `dst` (non-overlapping): a call's few
+/// arguments in straight-line code, without the unrolled loop's
+/// prologue and remainder.
+///
+/// # Safety
+///
+/// `src` holds `n` initialized objects the caller forgets; `dst` has room
+/// for `n`.
+#[inline(always)]
+unsafe fn move_objects(src: *const Object, dst: *mut Object, n: usize) {
+    // SAFETY: the caller's contract.
+    unsafe {
+        match n {
+            0 => {}
+            1 => dst.write(src.read()),
+            2 => {
+                dst.write(src.read());
+                dst.add(1).write(src.add(1).read());
+            }
+            3 => {
+                dst.write(src.read());
+                dst.add(1).write(src.add(1).read());
+                dst.add(2).write(src.add(2).read());
+            }
+            _ => std::ptr::copy_nonoverlapping(src, dst, n),
+        }
+    }
+}
+
+/// Write `n` unbound markers from `dst` (a frame's unset locals).
+///
+/// # Safety
+///
+/// `dst` has room for `n` objects, whose old contents need no drop.
+#[inline(always)]
+unsafe fn fill_unbound_raw(dst: *mut Object, n: usize) {
+    // SAFETY: the caller's contract; `Unbound` owns nothing.
+    unsafe {
+        match n {
+            0 => {}
+            1 => dst.write(Object::Unbound),
+            2 => {
+                dst.write(Object::Unbound);
+                dst.add(1).write(Object::Unbound);
+            }
+            _ => {
+                for k in 0..n {
+                    dst.add(k).write(Object::Unbound);
+                }
+            }
+        }
+    }
+}
+
 /// Extend `v` to `n` slots with `Unbound` (the fresh locals of an
 /// activation), in line: a call's handful of slots isn't worth
 /// `Vec::resize`'s out-of-line loop.

From 30ab7a3be7d04df0f9fe17d71c8442689d7cdb12 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 23:14:24 -0700
Subject: [PATCH 022/173] perf: native posixpath accelerators, cheaper
 struct_time and lazy frames

logging and the stdlib utility workloads spent much of their time in
code CPython runs natively:

- `posix._path_normpath` and `posix._path_splitroot_ex`, the C
  accelerators `posixpath` imports when present, now exist (with the
  pure-Python fallbacks' exact results, and leaf fast halves for `str`
  and `bytes`). `os.path.normpath` drops from ~19,000 to ~2,000
  instructions, and `pathlib` parses every path through `splitroot`.
- `time.localtime` builds its `struct_time` from a cached type and
  interned field keys whose hashes are kept (struct sequences in
  general, `os.stat` results included); `time.strftime` takes an ASCII
  format straight to libc. localtime 13,500 -> 9,700, strftime
  19,000 -> 13,900 instructions.
- `hasattr`/`getattr` of a module's global runs as a leaf (2,900 ->
  1,000).
- `sys._getframe` materializes only the frame it returns, linking
  `f_back` lazily to the caller's shell (the frames below are built
  only if something walks to them); `sys._getframemodulename` reads the
  shell's globals without building a frame object.
- A quiet return from an activation whose frame object exists settles
  that object without entering the general loop when nothing observes
  returns.

Suite (instructions vs. the branch base, pinned seed): logging_mix
-4.6%, stdlib_utils -3.7%, geomean 0.9948.
---
 crates/weavepy-vm/src/lib.rs                  |  94 +++++---
 crates/weavepy-vm/src/object.rs               |  78 ++++++-
 crates/weavepy-vm/src/stdlib/os.rs            | 203 ++++++++++++++++-
 .../weavepy-vm/src/stdlib/python/posix_mod.py |   5 +
 crates/weavepy-vm/src/stdlib/sys.rs           |  10 +-
 crates/weavepy-vm/src/stdlib/time.rs          |  56 +++--
 .../test_frame_and_path_accelerators.py       | 204 ++++++++++++++++++
 7 files changed, 598 insertions(+), 52 deletions(-)
 create mode 100644 tests/regrtest/test_frame_and_path_accelerators.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 2621f98f..7b045d20 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -8626,23 +8626,17 @@ impl Interpreter {
                 }
                 Ok(FrameOutcome::Returned(v))
             }
-            // A shell, but nothing the full epilogue would act on.
+            // A shell, and no observer for the general loop's return
+            // events: its epilogue, without entering it (see
+            // `Self::finish_quiet_return`).
             (
                 QuietExit::Outcome {
                     stepped: Ok(StepOutcome::Return(v)),
                     ..
                 },
                 Some(shell),
-            ) if !shell
-                .has_materialized
-                .load(std::sync::atomic::Ordering::Relaxed)
-                && self.exc_info_len() <= exc_depth =>
-            {
-                self.pop_frame_shell();
-                // A lazy traceback entry still holding the shell had its
-                // frame object built by the pop.
-                Self::settle_exited_frame_object(&shell, frame, true);
-                self.recycle_frame_shell(shell);
+            ) if self.return_settles_quietly(&shell, exc_depth) => {
+                self.finish_quiet_return(frame, shell, exc_depth);
                 Ok(FrameOutcome::Returned(v))
             }
             // An exception leaving an unobserved activation with no handler
@@ -10625,23 +10619,16 @@ impl Interpreter {
                 },
                 None,
             ) => FrameOutcome::Returned(v),
-            // A shell, but nothing the full epilogue would act on.
+            // A shell, and no observer for the general loop's return
+            // events: its epilogue, without entering it.
             (
                 QuietExit::Outcome {
                     stepped: Ok(StepOutcome::Return(v)),
                     ..
                 },
                 Some(shell),
-            ) if !shell
-                .has_materialized
-                .load(std::sync::atomic::Ordering::Relaxed)
-                && self.exc_info_len() <= exc_depth_on_entry =>
-            {
-                self.pop_frame_shell();
-                // A lazy traceback entry still holding the shell had its
-                // frame object built by the pop.
-                Self::settle_exited_frame_object(&shell, frame, true);
-                self.recycle_frame_shell(shell);
+            ) if self.return_settles_quietly(&shell, exc_depth_on_entry) => {
+                self.finish_quiet_return(frame, shell, exc_depth_on_entry);
                 FrameOutcome::Returned(v)
             }
             // Everything else finishes in the general loop, which takes
@@ -20760,12 +20747,12 @@ impl Interpreter {
         }
         // An activation whose shell the full epilogue would act on
         // finishes through `quiet_run`; so does a return into a caller
-        // whose protocol would yield straight away.
-        if top
-            .act
-            .shell
-            .as_deref()
-            .is_some_and(|s| !self.shell_pops_quietly(s, top.exc_depth))
+        // whose protocol would yield straight away. (A frame object alone
+        // settles in `retire_quiet_shell` while nothing observes returns.)
+        if top.act.shell.as_deref().is_some_and(|s| {
+            !self.shell_pops_quietly(s, top.exc_depth)
+                && (self.exc_info_len() > top.exc_depth || crate::trace::any_observers_active())
+        })
             || self.gil_countdown <= 2
             || crate::hot_gates::loop_gen() != snap_gen
         {
@@ -25758,6 +25745,19 @@ impl Interpreter {
             // A function's stored attribute (`__doc__` once read,
             // `__wrapped__`).
             Object::Function(f) => return function_stored_attr(f, name_obj).map(Ok),
+            // A plain module's global (`hasattr(os, 'getpid')`): its
+            // dictionary's value (`ModuleType`'s descriptors are dunders;
+            // a missing name may reach a module `__getattr__`).
+            Object::Module(m)
+                if !name.starts_with("__") && !crate::object::module_classes_possible() =>
+            {
+                return m
+                    .dict
+                    .borrow()
+                    .get(&DictKey(args[1].clone()))
+                    .cloned()
+                    .map(Ok);
+            }
             _ => return None,
         };
         // A dunder nothing supplies (`copy`'s `__deepcopy__` probe): the
@@ -31289,6 +31289,44 @@ impl Interpreter {
         }
     }
 
+    /// Whether a quiet loop's return from an activation with `shell` may
+    /// skip the general loop: nothing the full epilogue acts on (no frame
+    /// object, no handled-exception entries left above `exc_depth`), or
+    /// no observer for its `'return'` events (the epilogue's object
+    /// settling and stack reconciling then run here, see
+    /// [`Self::finish_quiet_return`]). A `sys._getframe` handle, or a
+    /// frame object some traceback entry built, no longer sends the
+    /// return through the general loop.
+    #[inline]
+    fn return_settles_quietly(&self, shell: &crate::object::FrameShell, exc_depth: usize) -> bool {
+        (!shell
+            .has_materialized
+            .load(std::sync::atomic::Ordering::Relaxed)
+            && self.exc_info_len() <= exc_depth)
+            || !crate::trace::any_observers_active()
+    }
+
+    /// `run_activation`'s epilogue for a normal return of an ordinary
+    /// activation that [`Self::return_settles_quietly`] accepted: pop its
+    /// shell, settle its frame object, drop the handled-exception entries
+    /// it left, park the shell.
+    #[inline(never)]
+    fn finish_quiet_return(
+        &mut self,
+        frame: &Frame,
+        shell: Rc<crate::object::FrameShell>,
+        exc_depth: usize,
+    ) {
+        self.pop_frame_shell();
+        // A lazy traceback entry still holding the shell had its frame
+        // object built by the pop.
+        Self::settle_exited_frame_object(&shell, frame, true);
+        if self.exc_info_len() > exc_depth {
+            self.exc_info_stack.borrow_mut().truncate(exc_depth);
+        }
+        self.recycle_frame_shell(shell);
+    }
+
     /// The general loop's unwind of an inline activation that `exc`
     /// leaves, when nothing observes the frame and no handler in it covers
     /// the raise (see `quiet_catch`): this frame's traceback entry, then
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index f15dda17..6b5f6b97 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -1352,6 +1352,67 @@ pub fn materialize_stack_at(stack: &FrameStack, idx: usize) -> Option<Rc<PyFrame
     back
 }
 
+/// [`materialize_stack_at`] for the one frame at `idx`: its `f_back` is
+/// its caller's frame object when one exists, else a lazy link to the
+/// caller's shell (see [`PyFrame::lazy_back`]), so the frames below are
+/// built only if something walks to them (`logging`'s `findCaller` reads
+/// two or three `f_back`s of a much deeper stack). A generator-family
+/// activation or caller, whose frame objects' links its resumes manage,
+/// takes the eager walk.
+pub fn materialize_stack_at_lazy(stack: &FrameStack, idx: usize) -> Option<Rc<PyFrame>> {
+    let s = stack.borrow();
+    let shell = s.get(idx)?;
+    let caller = idx.checked_sub(1).map(|i| &s[i]);
+    if shell.is_gen || caller.is_some_and(|c| c.is_gen) {
+        drop(s);
+        return materialize_stack_at(stack, idx);
+    }
+    let existing = shell.materialized.borrow().clone();
+    let mut refreshed = existing.is_some();
+    let py = match existing {
+        Some(py) => {
+            shell.refresh_materialized(&py);
+            py
+        }
+        // Counted as live on the stack, as the eager walk counts it.
+        None => shell.materialize(None),
+    };
+    match caller {
+        None => py.set_back(None),
+        Some(c) => {
+            let caller_py = c.materialized.borrow().clone();
+            match caller_py {
+                Some(back) => {
+                    c.refresh_materialized(&back);
+                    refreshed = true;
+                    let linked = py.lazy_back.borrow().is_none()
+                        && py.back.borrow().as_ref().is_some_and(|b| Rc::ptr_eq(b, &back));
+                    if !linked {
+                        py.set_back(Some(back));
+                    }
+                }
+                None => {
+                    let linked = py.back.borrow().is_none()
+                        && py
+                            .lazy_back
+                            .borrow()
+                            .as_ref()
+                            .is_some_and(|l| Rc::ptr_eq(&l.0, c));
+                    if !linked {
+                        *py.back.borrow_mut() = None;
+                        *py.lazy_back.borrow_mut() = Some(LazyShellRef::new(c.clone()));
+                    }
+                }
+            }
+        }
+    }
+    if refreshed {
+        // As in `materialize_stack_at`: a new holder of existing objects.
+        crate::hot_gates::bump_loop_gen();
+    }
+    Some(py)
+}
+
 /// Materialise and return the top frame of a stack, or `None` when
 /// the stack is empty.
 pub fn materialize_stack_top(stack: &FrameStack) -> Option<Rc<PyFrame>> {
@@ -1487,7 +1548,7 @@ pub(crate) fn materialize_shell(shell: &Rc<FrameShell>) -> Rc<PyFrame> {
             .iter()
             .rposition(|s| Rc::ptr_eq(s, shell));
         if let Some(idx) = idx {
-            if let Some(py) = materialize_stack_at(&handles.frame_stack, idx) {
+            if let Some(py) = materialize_stack_at_lazy(&handles.frame_stack, idx) {
                 return py;
             }
         }
@@ -1505,7 +1566,7 @@ pub(crate) fn materialize_shell(shell: &Rc<FrameShell>) -> Rc<PyFrame> {
             .ok()
             .and_then(|s| s.iter().rposition(|s| Rc::ptr_eq(s, shell)));
         if let Some(idx) = idx {
-            if let Some(py) = materialize_stack_at(&stack, idx) {
+            if let Some(py) = materialize_stack_at_lazy(&stack, idx) {
                 return py;
             }
         }
@@ -3189,9 +3250,22 @@ static MODULE_CLASS: std::sync::LazyLock<
 /// Record `m.__class__ = cls`. The caller validates that `cls` is a
 /// `ModuleType` subclass.
 pub fn set_module_class(m: &Rc<PyModule>, cls: Object) {
+    MODULE_CLASSES_SET.store(true, std::sync::atomic::Ordering::Release);
     MODULE_CLASS.write().insert(Rc::as_ptr(m) as usize, cls);
 }
 
+/// Whether any module ever got a class of its own (see
+/// [`set_module_class`]): until then every module's attributes are its
+/// dictionary's, `ModuleType`'s own descriptors aside.
+static MODULE_CLASSES_SET: std::sync::atomic::AtomicBool =
+    std::sync::atomic::AtomicBool::new(false);
+
+/// [`MODULE_CLASSES_SET`].
+#[inline]
+pub fn module_classes_possible() -> bool {
+    MODULE_CLASSES_SET.load(std::sync::atomic::Ordering::Acquire)
+}
+
 /// The class assigned via [`set_module_class`], if any. `None` means the
 /// stock `types.ModuleType`.
 pub fn module_class(m: &Rc<PyModule>) -> Option<Rc<crate::types::TypeObject>> {
diff --git a/crates/weavepy-vm/src/stdlib/os.rs b/crates/weavepy-vm/src/stdlib/os.rs
index 76c3870d..b9ba53b2 100644
--- a/crates/weavepy-vm/src/stdlib/os.rs
+++ b/crates/weavepy-vm/src/stdlib/os.rs
@@ -201,6 +201,24 @@ pub fn build(cache: &ModuleCache) -> Rc<PyModule> {
             crate::leaf_builtins::register(f);
         }
         d.insert(DictKey(Object::from_static("getpid")), getpid);
+        // CPython's C accelerators for `posixpath.splitroot` and
+        // `posixpath.normpath`, which `posixpath` imports when present
+        // (`pathlib` splits every path it parses through `splitroot`).
+        #[cfg(not(windows))]
+        for (name, body, fast) in [
+            (
+                "_path_splitroot_ex",
+                os_path_splitroot_ex as fn(&[Object]) -> Result<Object, RuntimeError>,
+                path_splitroot_fast as crate::leaf_builtins::Fast,
+            ),
+            ("_path_normpath", os_path_normpath, path_normpath_fast),
+        ] {
+            let f = builtin(name, body);
+            if let Object::Builtin(b) = &f {
+                crate::leaf_builtins::register_fast(b, fast);
+            }
+            d.insert(DictKey(Object::from_static(name)), f);
+        }
         // 3.14 `posix._is_inputhook_installed()` (gh-121886): `_pyrepl`'s
         // console polls it for `PyOS_InputHook`; WeavePy has no C input
         // hook, so it is always `False`.
@@ -7688,8 +7706,10 @@ pub(crate) fn struct_seq_instance(
     let inst = crate::types::PyInstance::new(ty);
     {
         let mut d = inst.dict_cell().borrow_mut();
+        // (Room for the hidden extras `time.struct_time` adds.)
+        d.reserve(fields.len() + 2);
         for (field, value) in fields.iter().zip(values.iter()) {
-            d.insert(DictKey(Object::from_static(field)), value.clone());
+            d.insert(static_name_key(field), value.clone());
         }
     }
     // Struct sequences subclass `tuple`, so give the instance a native tuple
@@ -7701,6 +7721,187 @@ pub(crate) fn struct_seq_instance(
     Object::Instance(Rc::new(inst))
 }
 
+/// The dictionary key for a static field name, built once per thread with
+/// its hash cached: a struct sequence's fields (an `os.stat` result, a
+/// `time.struct_time`) are stored and read without allocating or hashing
+/// their names each time.
+pub(crate) fn static_name_key(name: &'static str) -> DictKey {
+    thread_local! {
+        static KEYS: RefCell<std::collections::HashMap<(usize, usize), Object, crate::fasthash::FxBuildHasher>> =
+            RefCell::new(std::collections::HashMap::default());
+    }
+    KEYS.with(|keys| {
+        let mut keys = keys.borrow_mut();
+        let key = keys
+            .entry((name.as_ptr() as usize, name.len()))
+            .or_insert_with(|| {
+                let s = crate::shared_value::SharedStr::from(name);
+                crate::shared_value::SharedStr::hash_cached(&s);
+                Object::Str(s)
+            });
+        DictKey(key.clone())
+    })
+}
+
+/// The length of a POSIX path's root (`posixpath.splitroot`'s middle part):
+/// none for a relative path, one separator for an absolute one, and two
+/// for exactly two leading separators (implementation defined by POSIX).
+fn posix_root_len<T: PartialEq>(p: &[T], sep: &T) -> usize {
+    if p.first() != Some(sep) {
+        0
+    } else if p.get(1) != Some(sep) || p.get(2) == Some(sep) {
+        1
+    } else {
+        2
+    }
+}
+
+/// `posixpath.normpath`'s pure-Python algorithm over the units of `path`
+/// (bytes, UTF-8 bytes or code points): `A//B`, `A/./B` and `A/foo/../B`
+/// all become `A/B`.
+fn posix_normpath_units<T: PartialEq + Copy>(path: &[T], sep: T, dot: T) -> Vec<T> {
+    if path.is_empty() {
+        return vec![dot];
+    }
+    let root = posix_root_len(path, &sep);
+    let (initial, rest) = path.split_at(root);
+    let dotdot = [dot, dot];
+    let mut comps: Vec<&[T]> = Vec::new();
+    for comp in rest.split(|c| *c == sep) {
+        if comp.is_empty() || comp == [dot] {
+            continue;
+        }
+        if comp != dotdot
+            || (initial.is_empty() && comps.is_empty())
+            || comps.last().is_some_and(|l| *l == dotdot)
+        {
+            comps.push(comp);
+        } else {
+            comps.pop();
+        }
+    }
+    let mut out = initial.to_vec();
+    for (i, comp) in comps.iter().enumerate() {
+        if i > 0 {
+            out.push(sep);
+        }
+        out.extend_from_slice(comp);
+    }
+    if out.is_empty() {
+        out.push(dot);
+    }
+    out
+}
+
+/// A path argument of `func` as `str` or `bytes` (an `os.PathLike`'s
+/// `__fspath__()`), with CPython's `path_t` converter's message for
+/// anything else.
+fn path_arg(func: &str, p: &Object) -> Result<Object, RuntimeError> {
+    if matches!(p, Object::Instance(_)) {
+        return os_fspath(std::slice::from_ref(p));
+    }
+    if matches!(p, Object::Str(_) | Object::WStr(_) | Object::Bytes(_)) {
+        return Ok(p.clone());
+    }
+    Err(type_error(format!(
+        "{func}: path should be string, bytes or os.PathLike, not {}",
+        p.type_name()
+    )))
+}
+
+/// The leaf half of `_path_splitroot_ex` for a `str` or `bytes` path.
+fn path_splitroot_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
+    let r = match args {
+        [p @ Object::Str(s)] => {
+            let n = posix_root_len(s.as_bytes(), &b'/');
+            let (root, tail) = s.split_at(n);
+            Object::new_tuple_array([
+                Object::from_static(""),
+                Object::from_str(root.to_owned()),
+                if n == 0 { p.clone() } else { Object::from_str(tail.to_owned()) },
+            ])
+        }
+        [p @ Object::Bytes(b)] => {
+            let n = posix_root_len(b, &b'/');
+            let (root, tail) = b.split_at(n);
+            Object::new_tuple_array([
+                Object::new_bytes(Vec::new()),
+                Object::new_bytes(root.to_vec()),
+                if n == 0 { p.clone() } else { Object::new_bytes(tail.to_vec()) },
+            ])
+        }
+        _ => return None,
+    };
+    Some(Ok(r))
+}
+
+/// `posix._path_splitroot_ex(p)`: split a pathname into drive (always
+/// empty on POSIX), root and tail, as `posixpath`'s fallback does.
+fn os_path_splitroot_ex(args: &[Object]) -> Result<Object, RuntimeError> {
+    let [p] = args else {
+        return Err(type_error(format!(
+            "_path_splitroot_ex() takes exactly one argument ({} given)",
+            args.len()
+        )));
+    };
+    if let Some(r) = path_splitroot_fast(args) {
+        return r;
+    }
+    let p = path_arg("_path_splitroot_ex", p)?;
+    if let Object::WStr(cps) = &p {
+        let n = posix_root_len(cps, &u32::from(b'/'));
+        return Ok(Object::new_tuple_array([
+            Object::from_static(""),
+            Object::str_from_codepoints(cps[..n].to_vec()),
+            if n == 0 { p.clone() } else { Object::str_from_codepoints(cps[n..].to_vec()) },
+        ]));
+    }
+    path_splitroot_fast(std::slice::from_ref(&p))
+        .unwrap_or_else(|| Err(type_error("expected str, bytes or os.PathLike object")))
+}
+
+/// The leaf half of `_path_normpath` for a `str` or `bytes` path.
+fn path_normpath_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
+    let r = match args {
+        [Object::Str(s)] => {
+            // `/` and `.` are ASCII: splitting the UTF-8 bytes at them
+            // keeps every other character whole.
+            let out = posix_normpath_units(s.as_bytes(), b'/', b'.');
+            match String::from_utf8(out) {
+                Ok(out) => Object::from_str(out),
+                Err(_) => return None,
+            }
+        }
+        [Object::Bytes(b)] => Object::new_bytes(posix_normpath_units(b, b'/', b'.')),
+        _ => return None,
+    };
+    Some(Ok(r))
+}
+
+/// `posix._path_normpath(path)`: `posixpath.normpath`, as its pure-Python
+/// fallback computes it.
+fn os_path_normpath(args: &[Object]) -> Result<Object, RuntimeError> {
+    let [p] = args else {
+        return Err(type_error(format!(
+            "_path_normpath() takes exactly one argument ({} given)",
+            args.len()
+        )));
+    };
+    if let Some(r) = path_normpath_fast(args) {
+        return r;
+    }
+    let p = path_arg("_path_normpath", p)?;
+    if let Object::WStr(cps) = &p {
+        return Ok(Object::str_from_codepoints(posix_normpath_units(
+            cps,
+            u32::from(b'/'),
+            u32::from(b'.'),
+        )));
+    }
+    path_normpath_fast(std::slice::from_ref(&p))
+        .unwrap_or_else(|| Err(type_error("expected str, bytes or os.PathLike object")))
+}
+
 /// Construct an `os.terminal_size` instance with the given dimensions.
 fn make_terminal_size(columns: i64, lines: i64) -> Object {
     struct_seq_instance(
diff --git a/crates/weavepy-vm/src/stdlib/python/posix_mod.py b/crates/weavepy-vm/src/stdlib/python/posix_mod.py
index 65856df2..25d60693 100644
--- a/crates/weavepy-vm/src/stdlib/python/posix_mod.py
+++ b/crates/weavepy-vm/src/stdlib/python/posix_mod.py
@@ -34,6 +34,11 @@
 # 3.14: `_pyrepl`'s console polls `posix._is_inputhook_installed()` /
 # `nt._is_inputhook_installed()` for `PyOS_InputHook`.
 _is_inputhook_installed = _os._is_inputhook_installed
+# CPython's C accelerators for `posixpath.splitroot` and `posixpath.normpath`
+# (`posixpath` imports them from here when present).
+if hasattr(_os, "_path_normpath"):
+    _path_normpath = _os._path_normpath
+    _path_splitroot_ex = _os._path_splitroot_ex
 
 if hasattr(_os, "_fcopyfile"):
     _fcopyfile = _os._fcopyfile
diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs
index 466b10bc..2262e0df 100644
--- a/crates/weavepy-vm/src/stdlib/sys.rs
+++ b/crates/weavepy-vm/src/stdlib/sys.rs
@@ -1586,7 +1586,7 @@ fn sys_getframe(
     let idx = len - 1 - depth;
     // RFC 0058: the spine holds cheap shells; the Python-visible
     // frame object is materialised on demand right here.
-    match crate::object::materialize_stack_at(frame_stack, idx) {
+    match crate::object::materialize_stack_at_lazy(frame_stack, idx) {
         Some(py) => {
             // PEP 578: `sys._getframe` audits with the frame object.
             audit_event("sys._getframe", &[Object::Frame(py.clone())])?;
@@ -1622,11 +1622,9 @@ fn sys_getframemodulename(
         return Ok(Object::None);
     }
     let idx = len - 1 - depth;
-    let Some(py) = crate::object::materialize_stack_at(frame_stack, idx) else {
-        return Ok(Object::None);
-    };
-    let name = py
-        .globals
+    // The shell's globals are the frame's: no frame object needed.
+    let globals = frame_stack.borrow()[idx].globals.clone();
+    let name = globals
         .borrow()
         .get(&crate::object::StrKey("__name__"))
         .cloned();
diff --git a/crates/weavepy-vm/src/stdlib/time.rs b/crates/weavepy-vm/src/stdlib/time.rs
index 8bd8a3f0..51f64248 100644
--- a/crates/weavepy-vm/src/stdlib/time.rs
+++ b/crates/weavepy-vm/src/stdlib/time.rs
@@ -290,15 +290,27 @@ const STRUCT_TIME_FIELDS: [&str; 9] = [
 /// bare tuple (the old shape) broke them with `'tuple' object has no attribute
 /// 'tm_year'`.
 fn struct_time_type() -> Rc<crate::types::TypeObject> {
-    // Full CPython layout: 9 sequence slots plus the two hidden named
-    // members, so `n_fields` is 11 and an 10/11-element constructor sequence
-    // fills `tm_zone`/`tm_gmtoff` positionally (test_structseq).
-    let slots: Vec<Option<&'static str>> = STRUCT_TIME_FIELDS
-        .iter()
-        .map(|f| Some(*f))
-        .chain([Some("tm_zone"), Some("tm_gmtoff")])
-        .collect();
-    crate::stdlib::os::struct_seq_type_layout("struct_time", "time", slots, 9)
+    // (The registry memoises the type per thread too; this skips building
+    // the layout and the name lookup for every `localtime`.)
+    thread_local! {
+        static TYPE: std::cell::OnceCell<Rc<crate::types::TypeObject>> =
+            const { std::cell::OnceCell::new() };
+    }
+    TYPE.with(|t| {
+        t.get_or_init(|| {
+            // Full CPython layout: 9 sequence slots plus the two hidden
+            // named members, so `n_fields` is 11 and an 10/11-element
+            // constructor sequence fills `tm_zone`/`tm_gmtoff`
+            // positionally (test_structseq).
+            let slots: Vec<Option<&'static str>> = STRUCT_TIME_FIELDS
+                .iter()
+                .map(|f| Some(*f))
+                .chain([Some("tm_zone"), Some("tm_gmtoff")])
+                .collect();
+            crate::stdlib::os::struct_seq_type_layout("struct_time", "time", slots, 9)
+        })
+        .clone()
+    })
 }
 
 fn make_struct_time(values: Vec<Object>) -> Object {
@@ -691,7 +703,7 @@ fn gettmarg(arg: Option<&Object>, func: &str) -> Result<TmFields, RuntimeError>
             let mut v = Vec::with_capacity(9);
             for f in STRUCT_TIME_FIELDS {
                 v.push(
-                    d.get(&DictKey(Object::from_static(f)))
+                    d.get(&crate::stdlib::os::static_name_key(f))
                         .cloned()
                         .ok_or_else(illegal)?,
                 );
@@ -730,11 +742,11 @@ fn gettmarg(arg: Option<&Object>, func: &str) -> Result<TmFields, RuntimeError>
     let (zone, gmtoff) = match arg {
         Some(Object::Instance(inst)) => {
             let d = inst.dict_cell().borrow();
-            let zone = match d.get(&DictKey(Object::from_static("tm_zone"))) {
+            let zone = match d.get(&crate::stdlib::os::static_name_key("tm_zone")) {
                 Some(z @ (Object::Str(_) | Object::WStr(_))) => Some(z.to_str()),
                 _ => None,
             };
-            let gmtoff = match d.get(&DictKey(Object::from_static("tm_gmtoff"))) {
+            let gmtoff = match d.get(&crate::stdlib::os::static_name_key("tm_gmtoff")) {
                 Some(Object::Int(v)) => Some(*v),
                 _ => None,
             };
@@ -888,6 +900,10 @@ fn time_strftime(args: &[Object]) -> Result<Object, RuntimeError> {
     // code points into the PUA window — non-ASCII chars are copied through
     // verbatim (never handed to libc), then mapped back at the end.
     let cps = match args.first() {
+        // An ASCII format (the usual one) is its own bridge encoding.
+        Some(Object::Str(s)) if s.is_ascii() => {
+            return strftime_bridged(s.to_string(), args);
+        }
         Some(o @ (Object::Str(_) | Object::WStr(_))) => o.str_codepoints().unwrap_or_default(),
         Some(other) => {
             return Err(type_error(format!(
@@ -897,7 +913,11 @@ fn time_strftime(args: &[Object]) -> Result<Object, RuntimeError> {
         }
         None => return Err(type_error("strftime expects format string")),
     };
-    let fmt = crate::builtins::bridge_encode_cps(&cps);
+    strftime_bridged(crate::builtins::bridge_encode_cps(&cps), args)
+}
+
+/// `time.strftime` of the bridge-encoded format `fmt` (see `time_strftime`).
+fn strftime_bridged(fmt: String, args: &[Object]) -> Result<Object, RuntimeError> {
     let mut tm = match args.get(1) {
         None => localtime_now_tm()?,
         Some(o) => {
@@ -933,6 +953,12 @@ fn time_strftime(args: &[Object]) -> Result<Object, RuntimeError> {
         ctm.tm_zone = zone_c
             .as_ref()
             .map_or(std::ptr::null_mut(), |c| c.as_ptr().cast_mut());
+        if fmt.bytes().all(|b| (1..=0x7f).contains(&b)) {
+            // One ASCII run: the whole format goes through libc.
+            let out = strftime_chunk(&fmt, &ctm);
+            drop(zone_c);
+            return Ok(crate::builtins::bridge_to_object(&out));
+        }
         let chars: Vec<char> = fmt.chars().collect();
         let mut out = String::new();
         let mut i = 0;
@@ -1114,11 +1140,11 @@ fn with_tz_extras(obj: Object, gmtoff: i64, zone: &str) -> Object {
     if let Object::Instance(inst) = &obj {
         let mut d = inst.dict_cell().borrow_mut();
         d.insert(
-            DictKey(Object::from_static("tm_gmtoff")),
+            crate::stdlib::os::static_name_key("tm_gmtoff"),
             Object::Int(gmtoff),
         );
         d.insert(
-            DictKey(Object::from_static("tm_zone")),
+            crate::stdlib::os::static_name_key("tm_zone"),
             Object::from_str(zone.to_owned()),
         );
     }
diff --git a/tests/regrtest/test_frame_and_path_accelerators.py b/tests/regrtest/test_frame_and_path_accelerators.py
new file mode 100644
index 00000000..4ccce46c
--- /dev/null
+++ b/tests/regrtest/test_frame_and_path_accelerators.py
@@ -0,0 +1,204 @@
+"""`sys._getframe` materializes one frame and links its `f_back` chain
+lazily, a returning activation with a frame object settles without the
+general loop, and the native `posix._path_normpath`/`_path_splitroot_ex`,
+`time.localtime`/`strftime` and module `hasattr` paths keep CPython's
+observable behavior."""
+
+import os
+import posixpath
+import sys
+import time
+
+
+# --- Lazily linked frames ----------------------------------------------
+
+def walk(depth):
+    f = sys._getframe(depth)
+    out = []
+    while f is not None:
+        out.append((f.f_code.co_name, f.f_lineno))
+        f = f.f_back
+    return out
+
+
+def level1():
+    return walk(1)
+
+
+def level2():
+    return level1()
+
+
+def level3():
+    return level2()
+
+
+expected = None
+for i in range(500):
+    got = level3()
+    if expected is None:
+        expected = got
+    assert got == expected, (i, got, expected)
+assert [n for n, _ in expected[:3]] == ["level1", "level2", "level3"], expected
+
+
+def keep_caller():
+    return sys._getframe(1)
+
+
+def holder():
+    fr = keep_caller()
+    first = fr.f_lineno
+    x = 1
+    second = fr.f_lineno
+    back = fr.f_back
+    return first, second, back.f_code.co_name, fr.f_code.co_name, x
+
+
+for i in range(500):
+    first, second, back_name, name, _ = holder()
+    assert second == first + 2, (first, second)
+    assert back_name == "<module>" and name == "holder"
+
+
+def own_frame():
+    f = sys._getframe()
+    a = 1
+    line_a = f.f_lineno
+    b = 2
+    line_b = f.f_lineno
+    return line_a, line_b, f.f_back.f_code.co_name
+
+
+for i in range(500):
+    la, lb, caller = own_frame()
+    assert lb == la + 2 and caller == "<module>"
+
+
+def escaping():
+    def inner():
+        return sys._getframe(1)
+    local_value = 42
+    return inner()
+
+
+for i in range(300):
+    fr = escaping()
+    assert fr.f_code.co_name == "escaping"
+    assert fr.f_locals["local_value"] == 42
+    assert fr.f_back is not None and fr.f_back.f_code.co_name == "<module>"
+
+
+def callee_of_departed():
+    return sys._getframe(0)
+
+
+def departed_caller():
+    return callee_of_departed()
+
+
+for i in range(300):
+    fr = departed_caller()
+    # The caller left the stack while the callee's frame lived: its frame
+    # object was built on the way out.
+    assert fr.f_back.f_code.co_name == "departed_caller"
+    assert fr.f_back.f_back.f_code.co_name == "<module>"
+
+
+def gen_frames():
+    yield sys._getframe(0).f_back.f_code.co_name
+    yield sys._getframe(1).f_code.co_name
+
+
+def consume():
+    return list(gen_frames())
+
+
+for i in range(200):
+    assert consume() == ["consume", "consume"]
+
+assert sys._getframemodulename(0) == "__main__"
+
+
+# --- posixpath accelerators ---------------------------------------------
+
+def py_splitroot(p):
+    p = os.fspath(p)
+    sep, empty = (b"/", b"") if isinstance(p, bytes) else ("/", "")
+    if p[:1] != sep:
+        return empty, empty, p
+    elif p[1:2] != sep or p[2:3] == sep:
+        return empty, sep, p[1:]
+    else:
+        return empty, p[:2], p[2:]
+
+
+def py_normpath(path):
+    path = os.fspath(path)
+    if isinstance(path, bytes):
+        sep, dot, dotdot = b"/", b".", b".."
+    else:
+        sep, dot, dotdot = "/", ".", ".."
+    if not path:
+        return dot
+    _, initial_slashes, path = py_splitroot(path)
+    new_comps = []
+    for comp in path.split(sep):
+        if not comp or comp == dot:
+            continue
+        if (comp != dotdot or (not initial_slashes and not new_comps)
+                or (new_comps and new_comps[-1] == dotdot)):
+            new_comps.append(comp)
+        elif new_comps:
+            new_comps.pop()
+    path = initial_slashes + sep.join(new_comps)
+    return path or dot
+
+
+cases = ["", ".", "..", "/", "//", "///", "////a", "//a/b", "a/b/../..",
+         "../a", "/../a", "a//b/./c/..", "/a/./b/../c/5", "./", "a/b/",
+         "..//..", "/..", "x/../../y", "é/./ü/..", "a\x00b/../c",
+         "/a/\udcff/../b", "//\udcff", "a/../../b/./c//"]
+for c in cases:
+    assert posixpath.normpath(c) == py_normpath(c), c
+    assert posixpath.splitroot(c) == py_splitroot(c), c
+    if "\udcff" not in c:
+        b = c.encode()
+        assert posixpath.normpath(b) == py_normpath(b), c
+        assert posixpath.splitroot(b) == py_splitroot(b), c
+
+
+class P:
+    def __fspath__(self):
+        return "/a/./b/../c"
+
+
+assert posixpath.normpath(P()) == "/a/c"
+assert posixpath.splitroot(P()) == ("", "/", "a/./b/../c")
+for bad in (1, None, [1]):
+    for fn in (posixpath.normpath, posixpath.splitroot):
+        try:
+            fn(bad)
+        except TypeError:
+            pass
+        else:
+            raise AssertionError((fn, bad))
+
+
+# --- time.struct_time and strftime ------------------------------------
+
+t = time.localtime(1700000000.5)
+assert len(t) == 9 and tuple(t)[:3] == (t.tm_year, t.tm_mon, t.tm_mday)
+assert isinstance(t.tm_zone, str) and isinstance(t.tm_gmtoff, int)
+assert time.strftime("%Y", t) == str(t.tm_year)
+assert time.strftime("", t) == ""
+assert time.strftime("café %Y", t) == "café " + str(t.tm_year)
+assert time.strftime("%Y-%m-%d", t) == "%04d-%02d-%02d" % (t.tm_year, t.tm_mon, t.tm_mday)
+assert time.mktime(t) == 1700000000.0
+
+
+# --- hasattr / getattr on modules -------------------------------------
+
+assert hasattr(os, "getpid") and not hasattr(os, "no_such_name")
+assert getattr(os, "sep") == "/" and getattr(os, "no_such_name", 5) == 5
+assert hasattr(os, "__dict__") and getattr(os, "__name__") == "os"

From 888840111380f139150a4de4fdeb1fbbfb05eb67 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 00:04:36 -0700
Subject: [PATCH 023/173] perf: render plain %-formats in one pass

`str % args` went through the general printf engine even for the plain
shapes logging and EAFP messages use (`"%s,%03d"` aside): every
conversion parsed into owned flag, width and precision strings, and a
mapping format looked each key up twice (once to vet the value, once to
render it), allocating and hashing the key both times.

A template whose every conversion is a bare `%s`, `%d` or `%i` of an
exact `str`, `int`, `bool` or `None` (by position, consuming every
argument, or by `%(key)` from an exact dict) now renders in one pass;
any other shape, a missing key or a leftover argument takes the general
path, which renders or raises as before. Struct sequences also fetch
their interned field keys once per instance instead of once per field.

`logging`'s `"%(asctime)s %(levelname)s %(name)s: %(message)s" % d`:
6,300 -> 3,200 instructions (CPython 5,700). Suite vs. the previous
commit (pinned seed): logging_mix -3.9%, exceptions -2.0%, geomean
0.9988.
---
 crates/weavepy-vm/src/lib.rs                  | 100 ++++++++++++++++++
 crates/weavepy-vm/src/stdlib/os.rs            |  21 +++-
 crates/weavepy-vm/src/stdlib/time.rs          |   9 +-
 .../test_frame_and_path_accelerators.py       |  27 +++++
 4 files changed, 149 insertions(+), 8 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 7b045d20..f1ad8c66 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -44385,6 +44385,11 @@ impl Interpreter {
         b: &Object,
         globals: &Rc<RefCell<DictData>>,
     ) -> Result<Object, RuntimeError> {
+        if let Object::Str(t) = a {
+            if let Some(out) = percent_render_plain_value(t, b) {
+                return Ok(Object::from_str(out));
+            }
+        }
         if a.is_str() {
             // Surrogate-aware printf: a `WStr` template (and any `WStr`
             // argument resolved via `%s`) bridges its lone surrogates into
@@ -68244,6 +68249,11 @@ fn percent_format_leaf_mapping(template: &str, value: &Object) -> Option<String>
     };
     let bytes = template.as_bytes();
     let map = d.try_borrow().ok()?;
+    // The plain shape (`logging`'s `%(asctime)s %(levelname)s ...`), in
+    // one pass.
+    if let Some(out) = percent_render_plain(template, PercentArgs::Mapping(&map)) {
+        return Some(out);
+    }
     let mut i = 0;
     while let Some(off) = bytes[i..].iter().position(|&b| b == b'%') {
         i += off + 1;
@@ -68286,7 +68296,97 @@ fn percent_format_leaf_mapping(template: &str, value: &Object) -> Option<String>
     percent_format(template, value).ok()
 }
 
+/// The arguments [`percent_render_plain`] renders from.
+enum PercentArgs<'a> {
+    Positional(&'a [Object]),
+    Mapping(&'a DictData),
+}
+
+/// `template % args` in one pass when every conversion is a plain `%s`,
+/// `%d` or `%i` (no flags, width or precision; a mapping key for a
+/// mapping, none for positional arguments) of an exact `str`, `int`,
+/// `bool` or `None` (`%s` only for `str`, `None`), and every positional
+/// argument is consumed: the text `percent_format_with` renders for that
+/// shape. `None` for anything else (the general path then renders it, or
+/// raises), including a mapping key the dictionary lacks.
+fn percent_render_plain(template: &str, args: PercentArgs<'_>) -> Option<String> {
+    use std::fmt::Write as _;
+    let bytes = template.as_bytes();
+    let mut out = String::with_capacity(template.len() + 32);
+    let (mut i, mut lit, mut next) = (0, 0, 0);
+    while let Some(off) = memchr::memchr(b'%', &bytes[i..]) {
+        let at = i + off;
+        out.push_str(&template[lit..at]);
+        let mut k = at + 1;
+        let v = match (bytes.get(k)?, &args) {
+            (b'%', _) => {
+                out.push('%');
+                i = k + 1;
+                lit = i;
+                continue;
+            }
+            (b'(', PercentArgs::Mapping(map)) => {
+                let close = k + 1 + memchr::memchr(b')', &bytes[k + 1..])?;
+                let key = &template[k + 1..close];
+                if key.contains('(') {
+                    return None;
+                }
+                k = close + 1;
+                map.get(&crate::object::StrKey(key))?
+            }
+            (_, PercentArgs::Positional(items)) => {
+                let v = items.get(next)?;
+                next += 1;
+                v
+            }
+            _ => return None,
+        };
+        match (bytes.get(k)?, v) {
+            (b's', Object::Str(s)) => out.push_str(s),
+            (b's' | b'd' | b'i', Object::Int(n)) => {
+                let _ = write!(out, "{n}");
+            }
+            (b's', Object::Bool(b)) => out.push_str(if *b { "True" } else { "False" }),
+            (b'd' | b'i', Object::Bool(b)) => out.push(if *b { '1' } else { '0' }),
+            (b's', Object::None) => out.push_str("None"),
+            _ => return None,
+        }
+        i = k + 1;
+        lit = i;
+    }
+    if let PercentArgs::Positional(items) = args {
+        if next != items.len() {
+            return None;
+        }
+    }
+    out.push_str(&template[lit..]);
+    Some(out)
+}
+
+/// [`percent_render_plain`] of `template % value` for an argument tuple,
+/// a lone scalar argument or an exact dict.
+fn percent_render_plain_value(template: &str, value: &Object) -> Option<String> {
+    let one;
+    let items: &[Object] = match value {
+        Object::Tuple(items) if !items.is_empty() => items,
+        Object::Str(_) | Object::Int(_) | Object::Bool(_) | Object::None => {
+            one = [value.clone()];
+            &one
+        }
+        Object::Dict(d) => {
+            let map = d.try_borrow().ok()?;
+            return percent_render_plain(template, PercentArgs::Mapping(&map));
+        }
+        _ => return None,
+    };
+    percent_render_plain(template, PercentArgs::Positional(items))
+}
+
 pub(crate) fn percent_format(template: &str, value: &Object) -> Result<String, RuntimeError> {
+    // The plain shape (`"warning number %d of %s" % (i, s)`).
+    if let Some(out) = percent_render_plain_value(template, value) {
+        return Ok(out);
+    }
     let mut noop = |_: &Object, _: char| Ok(None);
     percent_format_with(template, value, PercentMode::Str, &mut noop)
 }
diff --git a/crates/weavepy-vm/src/stdlib/os.rs b/crates/weavepy-vm/src/stdlib/os.rs
index b9ba53b2..f7a9cbe3 100644
--- a/crates/weavepy-vm/src/stdlib/os.rs
+++ b/crates/weavepy-vm/src/stdlib/os.rs
@@ -7708,8 +7708,9 @@ pub(crate) fn struct_seq_instance(
         let mut d = inst.dict_cell().borrow_mut();
         // (Room for the hidden extras `time.struct_time` adds.)
         d.reserve(fields.len() + 2);
-        for (field, value) in fields.iter().zip(values.iter()) {
-            d.insert(static_name_key(field), value.clone());
+        let keys = static_name_keys(fields);
+        for (key, value) in keys.iter().zip(values.iter()) {
+            d.insert(key.clone(), value.clone());
         }
     }
     // Struct sequences subclass `tuple`, so give the instance a native tuple
@@ -7721,6 +7722,22 @@ pub(crate) fn struct_seq_instance(
     Object::Instance(Rc::new(inst))
 }
 
+/// [`static_name_key`] for each of a struct sequence's `fields`, looked up
+/// once per instance.
+pub(crate) fn static_name_keys(fields: &'static [&'static str]) -> std::rc::Rc<[DictKey]> {
+    thread_local! {
+        static KEYS: RefCell<std::collections::HashMap<(usize, usize), std::rc::Rc<[DictKey]>, crate::fasthash::FxBuildHasher>> =
+            RefCell::new(std::collections::HashMap::default());
+    }
+    let id = (fields.as_ptr() as usize, fields.len());
+    if let Some(keys) = KEYS.with(|k| k.borrow().get(&id).cloned()) {
+        return keys;
+    }
+    let keys: std::rc::Rc<[DictKey]> = fields.iter().map(|f| static_name_key(f)).collect();
+    KEYS.with(|k| k.borrow_mut().insert(id, keys.clone()));
+    keys
+}
+
 /// The dictionary key for a static field name, built once per thread with
 /// its hash cached: a struct sequence's fields (an `os.stat` result, a
 /// `time.struct_time`) are stored and read without allocating or hashing
diff --git a/crates/weavepy-vm/src/stdlib/time.rs b/crates/weavepy-vm/src/stdlib/time.rs
index 51f64248..9d508193 100644
--- a/crates/weavepy-vm/src/stdlib/time.rs
+++ b/crates/weavepy-vm/src/stdlib/time.rs
@@ -700,13 +700,10 @@ fn gettmarg(arg: Option<&Object>, func: &str) -> Result<TmFields, RuntimeError>
         Some(Object::Tuple(t)) => t.to_vec(),
         Some(Object::Instance(inst)) => {
             let d = inst.dict_cell().borrow();
+            let keys = crate::stdlib::os::static_name_keys(&STRUCT_TIME_FIELDS);
             let mut v = Vec::with_capacity(9);
-            for f in STRUCT_TIME_FIELDS {
-                v.push(
-                    d.get(&crate::stdlib::os::static_name_key(f))
-                        .cloned()
-                        .ok_or_else(illegal)?,
-                );
+            for key in keys.iter() {
+                v.push(d.get(key).cloned().ok_or_else(illegal)?);
             }
             v
         }
diff --git a/tests/regrtest/test_frame_and_path_accelerators.py b/tests/regrtest/test_frame_and_path_accelerators.py
index 4ccce46c..6bc08dfb 100644
--- a/tests/regrtest/test_frame_and_path_accelerators.py
+++ b/tests/regrtest/test_frame_and_path_accelerators.py
@@ -202,3 +202,30 @@ def __fspath__(self):
 assert hasattr(os, "getpid") and not hasattr(os, "no_such_name")
 assert getattr(os, "sep") == "/" and getattr(os, "no_such_name", 5) == 5
 assert hasattr(os, "__dict__") and getattr(os, "__name__") == "os"
+
+
+# --- Plain %-formatting -------------------------------------------------
+
+assert "%s %s" % ("a", 1) == "a 1"
+assert "%s-%d-%i" % ("x", 2, 3) == "x-2-3"
+assert "a%%b%s" % ("c",) == "a%bc"
+assert "%s|%d|%s" % (True, True, None) == "True|1|None"
+assert "%(a)s %(b)d" % {"a": "x", "b": 2} == "x 2"
+assert "%(a(b))s" % {"a(b)": 1} == "1"
+assert "%s" % {"a": 1} == "{'a': 1}"
+assert "%(x)i%%" % {"x": False} == "0%"
+assert "%5s|%-3d|" % ("a", 4) == "    a|4  |"
+assert "%s" % (2 ** 70,) == str(2 ** 70)
+for template, args, exc in (("%s", ("a", "b"), TypeError), ("%s %s", ("a",), TypeError),
+                            ("%d", "x", TypeError), ("%(a)s", {}, KeyError),
+                            ("%", (), ValueError)):
+    try:
+        template % args
+    except exc:
+        pass
+    else:
+        raise AssertionError((template, args))
+record = {"asctime": "t", "levelname": "WARNING", "name": "n", "message": "m", "extra": 1}
+for i in range(2000):
+    assert "%(asctime)s %(levelname)s %(name)s: %(message)s" % record == "t WARNING n: m"
+    assert "%s/%d" % ("p", i) == "p/" + str(i)

From b570eed56a68c13430d746786aa59dc2a27e162c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 01:08:22 -0700
Subject: [PATCH 024/173] fix: report a tier-2 caller's call line to
 introspection

When tier 2 ran a framed activation natively, a call it made into the
interpreter left the activation's frame shell at its entry position, so
a callee reading f_back.f_lineno (logging's findCaller, warnings,
traceback.walk_stack) saw the caller's def line on the first native run.
The native code already stores the call site's position before its call
helpers; the helper now records it in the activation's shell, the
spine's top, as the interpreter's own call does.
---
 crates/weavepy-vm/src/lib.rs               |  3 +--
 crates/weavepy-vm/src/object.rs            |  6 ++++-
 crates/weavepy-vm/src/stdlib/os.rs         | 18 ++++++++++++---
 crates/weavepy-vm/src/tier2.rs             | 11 +++++++++
 tests/regrtest/test_tier2_caller_lineno.py | 26 ++++++++++++++++++++++
 5 files changed, 58 insertions(+), 6 deletions(-)
 create mode 100644 tests/regrtest/test_tier2_caller_lineno.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index f1ad8c66..bd766105 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -20752,8 +20752,7 @@ impl Interpreter {
         if top.act.shell.as_deref().is_some_and(|s| {
             !self.shell_pops_quietly(s, top.exc_depth)
                 && (self.exc_info_len() > top.exc_depth || crate::trace::any_observers_active())
-        })
-            || self.gil_countdown <= 2
+        }) || self.gil_countdown <= 2
             || crate::hot_gates::loop_gen() != snap_gen
         {
             return false;
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 6b5f6b97..3c21ca35 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -1386,7 +1386,11 @@ pub fn materialize_stack_at_lazy(stack: &FrameStack, idx: usize) -> Option<Rc<Py
                     c.refresh_materialized(&back);
                     refreshed = true;
                     let linked = py.lazy_back.borrow().is_none()
-                        && py.back.borrow().as_ref().is_some_and(|b| Rc::ptr_eq(b, &back));
+                        && py
+                            .back
+                            .borrow()
+                            .as_ref()
+                            .is_some_and(|b| Rc::ptr_eq(b, &back));
                     if !linked {
                         py.set_back(Some(back));
                     }
diff --git a/crates/weavepy-vm/src/stdlib/os.rs b/crates/weavepy-vm/src/stdlib/os.rs
index f7a9cbe3..72c49fe1 100644
--- a/crates/weavepy-vm/src/stdlib/os.rs
+++ b/crates/weavepy-vm/src/stdlib/os.rs
@@ -7835,7 +7835,11 @@ fn path_splitroot_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>>
             Object::new_tuple_array([
                 Object::from_static(""),
                 Object::from_str(root.to_owned()),
-                if n == 0 { p.clone() } else { Object::from_str(tail.to_owned()) },
+                if n == 0 {
+                    p.clone()
+                } else {
+                    Object::from_str(tail.to_owned())
+                },
             ])
         }
         [p @ Object::Bytes(b)] => {
@@ -7844,7 +7848,11 @@ fn path_splitroot_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>>
             Object::new_tuple_array([
                 Object::new_bytes(Vec::new()),
                 Object::new_bytes(root.to_vec()),
-                if n == 0 { p.clone() } else { Object::new_bytes(tail.to_vec()) },
+                if n == 0 {
+                    p.clone()
+                } else {
+                    Object::new_bytes(tail.to_vec())
+                },
             ])
         }
         _ => return None,
@@ -7870,7 +7878,11 @@ fn os_path_splitroot_ex(args: &[Object]) -> Result<Object, RuntimeError> {
         return Ok(Object::new_tuple_array([
             Object::from_static(""),
             Object::str_from_codepoints(cps[..n].to_vec()),
-            if n == 0 { p.clone() } else { Object::str_from_codepoints(cps[n..].to_vec()) },
+            if n == 0 {
+                p.clone()
+            } else {
+                Object::str_from_codepoints(cps[n..].to_vec())
+            },
         ]));
     }
     path_splitroot_fast(std::slice::from_ref(&p))
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index eac16025..741c5c77 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -6930,6 +6930,17 @@ fn call_with_activation_shell<T>(
     f: impl FnOnce(&mut super::Interpreter) -> T,
 ) -> T {
     let Some(code) = &ctx.frameless_code else {
+        // A framed activation's shell is the spine's top while its native
+        // code runs: the call site becomes its `f_lineno`, as the
+        // interpreter's own call would record it.
+        if let Ok(stack) = interp.frame_stack.try_borrow() {
+            if let Some(top) = stack.last() {
+                if std::ptr::eq(Rc::as_ptr(&*top.code), ctx.code_ptr) {
+                    top.lasti
+                        .store(jf.deopt_pc, std::sync::atomic::Ordering::Relaxed);
+                }
+            }
+        }
         return f(interp);
     };
     let shell_locals = Rc::new(GilRefCell::new(Vec::new()));
diff --git a/tests/regrtest/test_tier2_caller_lineno.py b/tests/regrtest/test_tier2_caller_lineno.py
new file mode 100644
index 00000000..2a5b1747
--- /dev/null
+++ b/tests/regrtest/test_tier2_caller_lineno.py
@@ -0,0 +1,26 @@
+# A caller's f_lineno, read through f_back while it runs a call, is the
+# call's line on every call, including the one on which tier 2 first
+# runs the caller natively (it used to report the caller's `def` line).
+
+import sys
+
+
+def callee():
+    f = sys._getframe()
+    return f.f_lineno, f.f_back.f_lineno
+
+
+def caller():
+    r = callee()
+    return r
+
+
+def run(n=3000):
+    return [caller() for _ in range(n)]
+
+
+res = run()
+want = (10, 14)
+bad = [(i, r) for i, r in enumerate(res) if r != want]
+assert not bad, bad[:5]
+print("test_tier2_caller_lineno: OK")

From b03ed7416c4879f5aaca2c67d784a754bd9962fa Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 23:10:57 -0700
Subject: [PATCH 025/173] perf: look up asyncio loop attributes by remembered
 position

The native call_soon, create_task and create_future check a stock
loop's _debug, _closed, _ready, call_soon, get_debug and _task_factory
on every call (about 500 reads per async_tree iteration). Each read
scanned the class's shared names linearly, and the module checks
(events.Handle, tasks.Task, futures.Future) hashed their names each
time. The reads now remember the name's position in the shared-names
table (verified on every hit, so a reused table address only costs a
search), and the module checks use precomputed hashes.

A task whose coroutine yields None (asyncio.sleep(0)) now reschedules
before looking for _asyncio_future_blocking, as the C Task does; the
failed lookup on None built an AttributeError every time.
is_native_future checks the exact native classes before walking the
MRO.

async_tree: 1771M -> 1704M instructions per call (-3.8%); sleep(0)
round trip -13%. Suite geomean 0.998x.
---
 crates/weavepy-vm/src/inst_dict.rs            |  51 +++++++
 .../weavepy-vm/src/stdlib/asyncio_events.rs   | 114 +++++++++++-----
 crates/weavepy-vm/src/stdlib/asyncio_mod.rs   |  29 ++--
 tests/regrtest/test_asyncio_loop_attr_memo.py | 126 ++++++++++++++++++
 4 files changed, 277 insertions(+), 43 deletions(-)
 create mode 100644 tests/regrtest/test_asyncio_loop_attr_memo.py

diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index 457fd71e..3b431051 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -226,6 +226,24 @@ unsafe impl Sync for NameMemo {}
 // SAFETY: as above.
 unsafe impl Send for NameMemo {}
 
+/// A name's position in the names table it was last found in (see
+/// [`SplitValues::get_memo`]). A hit is verified against the table, so a
+/// stale or torn pair only costs a search.
+#[derive(Default, Debug)]
+pub struct PosMemo {
+    keys: AtomicUsize,
+    pos: AtomicUsize,
+}
+
+impl PosMemo {
+    pub const fn new() -> Self {
+        Self {
+            keys: AtomicUsize::new(0),
+            pos: AtomicUsize::new(0),
+        }
+    }
+}
+
 /// The header of an instance's split values allocation; the values
 /// follow it.
 #[repr(C)]
@@ -625,6 +643,39 @@ impl SplitValues {
         n > before
     }
 
+    /// The value of attribute `name` (Python hash `hash`), with `memo`
+    /// remembering the names table it was last found in and its position
+    /// there. A name never moves within a table, so a hit checks just
+    /// that position (the name there, in case the table's address was
+    /// reused); anything else searches and remembers.
+    #[inline]
+    pub fn get_memo(
+        &self,
+        name: &str,
+        hash: i64,
+        memo: &PosMemo,
+    ) -> Option<&Object> {
+        let keys = self.keys()?;
+        let n = self.len();
+        let at = std::ptr::from_ref(keys) as usize;
+        let (k, i) = (
+            memo.keys.load(Ordering::Relaxed),
+            memo.pos.load(Ordering::Relaxed),
+        );
+        if k == at && i < n {
+            // SAFETY: slot `i` is published (below the values' length,
+            // which never exceeds the table's).
+            let h = unsafe { *keys.hashes[i].get() };
+            if h == hash && keys.get(i).is_some_and(|key| key_names_str(key, name)) {
+                return self.values().get(i);
+            }
+        }
+        let i = keys.position_hashed(n, name, hash)?;
+        memo.keys.store(at, Ordering::Relaxed);
+        memo.pos.store(i, Ordering::Relaxed);
+        self.values().get(i)
+    }
+
     /// [`Self::position`] for a borrowed name (identity of the bytes
     /// first: a name read off an interned string settles without a
     /// comparison).
diff --git a/crates/weavepy-vm/src/stdlib/asyncio_events.rs b/crates/weavepy-vm/src/stdlib/asyncio_events.rs
index 452679db..b429e8d0 100644
--- a/crates/weavepy-vm/src/stdlib/asyncio_events.rs
+++ b/crates/weavepy-vm/src/stdlib/asyncio_events.rs
@@ -198,40 +198,74 @@ fn install_handle(args: &[Object]) -> Result<Object, RuntimeError> {
 // ---------------------------------------------------------------------
 // BaseEventLoop.call_soon.
 
-/// The Python hashes of the instance attributes the fast paths read.
+/// A name the fast paths look up: its Python hash, and where an instance
+/// of the loop class last held it (see [`crate::inst_dict::PosMemo`]).
+struct AttrName {
+    name: &'static str,
+    hash: i64,
+    memo: crate::inst_dict::PosMemo,
+}
+
+impl AttrName {
+    fn new(name: &'static str) -> Self {
+        Self {
+            name,
+            hash: crate::object::py_str_hash(name),
+            memo: crate::inst_dict::PosMemo::new(),
+        }
+    }
+}
+
+/// The instance attributes the fast paths read, and the module globals
+/// they check.
 struct Names {
-    debug: i64,
-    closed: i64,
-    ready: i64,
-    call_soon: i64,
-    get_debug: i64,
-    task_factory: i64,
+    debug: AttrName,
+    closed: AttrName,
+    ready: AttrName,
+    call_soon: AttrName,
+    get_debug: AttrName,
+    task_factory: AttrName,
+    events: AttrName,
+    handle: AttrName,
+    tasks: AttrName,
+    task: AttrName,
+    futures: AttrName,
+    future: AttrName,
 }
 
 fn names() -> &'static Names {
     static NAMES: std::sync::OnceLock<Names> = std::sync::OnceLock::new();
     NAMES.get_or_init(|| Names {
-        debug: crate::object::py_str_hash("_debug"),
-        closed: crate::object::py_str_hash("_closed"),
-        ready: crate::object::py_str_hash("_ready"),
-        call_soon: crate::object::py_str_hash("call_soon"),
-        get_debug: crate::object::py_str_hash("get_debug"),
-        task_factory: crate::object::py_str_hash("_task_factory"),
+        debug: AttrName::new("_debug"),
+        closed: AttrName::new("_closed"),
+        ready: AttrName::new("_ready"),
+        call_soon: AttrName::new("call_soon"),
+        get_debug: AttrName::new("get_debug"),
+        task_factory: AttrName::new("_task_factory"),
+        events: AttrName::new("events"),
+        handle: AttrName::new("Handle"),
+        tasks: AttrName::new("tasks"),
+        task: AttrName::new("Task"),
+        futures: AttrName::new("futures"),
+        future: AttrName::new("Future"),
     })
 }
 
-/// Instance attribute `name` (Python hash `hash`), read without running
-/// code (a split layout answers a never-set name from its filter).
-fn inst_attr(inst: &PyInstance, name: &str, hash: i64) -> Option<Object> {
+/// Instance attribute `n`, read without running code (a split layout
+/// answers a never-set name from its filter, and a set one from the
+/// position it last had).
+fn inst_attr(inst: &PyInstance, n: &AttrName) -> Option<Object> {
     match inst.dict.published() {
         Some(d) => d
             .borrow()
-            .get(&crate::object::StrKeyHashed { s: name, hash })
+            .get(&crate::object::StrKeyHashed {
+                s: n.name,
+                hash: n.hash,
+            })
             .cloned(),
         None => {
             let split = inst.dict.split_cell().borrow();
-            let i = split.position_hashed(name, hash)?;
-            split.values().get(i).cloned()
+            split.get_memo(n.name, n.hash, &n.memo).cloned()
         }
     }
 }
@@ -271,7 +305,7 @@ fn resolves_to(cls: &TypeObject, name: &str, expected: &Object) -> bool {
 
 /// The `_ready` deque of a stock loop, if its `append` is the native one.
 fn ready_deque(inst: &PyInstance) -> Option<Object> {
-    let ready = inst_attr(inst, "_ready", names().ready)?;
+    let ready = inst_attr(inst, &names().ready)?;
     let Object::Instance(r) = &ready else {
         return None;
     };
@@ -304,7 +338,7 @@ fn fast_call_soon(
     // The class the loop's module would build (`events.Handle`, looked up
     // at call time: it may have been replaced, or come from another
     // import of `asyncio.events` than the one `hs` describes).
-    let current = global_module_attr(&ls.globals, "events", "Handle")
+    let current = global_module_attr(&ls.globals, &names().events, &names().handle)
         .is_some_and(|h| matches!(h, Object::Type(t) if Rc::ptr_eq(&t, &hs.cls)));
     if !current {
         return Ok(None);
@@ -364,7 +398,7 @@ pub(crate) fn call_soon(
             ls.call_soon_native
                 .get()
                 .is_some_and(|n| resolves_to(&li.cls(), "call_soon", n))
-        }) && inst_attr(li, "call_soon", names().call_soon).is_none();
+        }) && inst_attr(li, &names().call_soon).is_none();
         if native {
             if let Some(h) = fast_call_soon(ls, loop_, &cb, cb_args, &ctx)? {
                 return Ok(h);
@@ -391,9 +425,9 @@ pub(crate) fn get_debug(loop_: &Object) -> Result<bool, RuntimeError> {
     if let (Some(ls), Object::Instance(li)) = (&ls, loop_) {
         let native = verified(&NATIVE_GET_DEBUG, &li.class.borrow(), || {
             resolves_to(&li.cls(), "get_debug", &ls.get_debug)
-        }) && inst_attr(li, "get_debug", names().get_debug).is_none();
+        }) && inst_attr(li, &names().get_debug).is_none();
         if native {
-            if let Some(Object::Bool(b)) = inst_attr(li, "_debug", names().debug) {
+            if let Some(Object::Bool(b)) = inst_attr(li, &names().debug) {
                 return Ok(b);
             }
         }
@@ -418,24 +452,38 @@ fn plain_loop(ls: &LoopState, loop_: &Object) -> bool {
     });
     let n = names();
     stock
-        && matches!(inst_attr(li, "_debug", n.debug), Some(Object::Bool(false)))
+        && matches!(inst_attr(li, &n.debug), Some(Object::Bool(false)))
         && matches!(
-            inst_attr(li, "_closed", n.closed),
+            inst_attr(li, &n.closed),
             Some(Object::Bool(false))
         )
-        && inst_attr(li, "get_debug", n.get_debug).is_none()
+        && inst_attr(li, &n.get_debug).is_none()
 }
 
 /// `module.attr`, for the module bound to global `module` in `globals`.
-fn global_module_attr(globals: &Rc<RefCell<DictData>>, module: &str, attr: &str) -> Option<Object> {
+fn global_module_attr(
+    globals: &Rc<RefCell<DictData>>,
+    module: &AttrName,
+    attr: &AttrName,
+) -> Option<Object> {
     let m = globals
         .borrow()
-        .get(&crate::object::StrKey(module))
+        .get(&crate::object::StrKeyHashed {
+            s: module.name,
+            hash: module.hash,
+        })
         .cloned()?;
     let Object::Module(m) = m else {
         return None;
     };
-    let v = m.dict.borrow().get(&crate::object::StrKey(attr)).cloned();
+    let v = m
+        .dict
+        .borrow()
+        .get(&crate::object::StrKeyHashed {
+            s: attr.name,
+            hash: attr.hash,
+        })
+        .cloned();
     v
 }
 
@@ -454,10 +502,10 @@ fn loop_create_task_method(
                 unreachable!("plain_loop checks for an instance")
             };
             let no_factory = matches!(
-                inst_attr(li, "_task_factory", names().task_factory),
+                inst_attr(li, &names().task_factory),
                 Some(Object::None)
             );
-            let native = global_module_attr(&ls.globals, "tasks", "Task")
+            let native = global_module_attr(&ls.globals, &names().tasks, &names().task)
                 .is_some_and(|t| crate::stdlib::asyncio_mod::is_task_class(&t));
             if no_factory && native {
                 return crate::stdlib::asyncio_mod::new_task(loop_, coro.clone(), kwargs);
@@ -470,7 +518,7 @@ fn loop_create_task_method(
 /// `BaseEventLoop.create_future(self)`.
 fn loop_create_future_method(ls: &LoopState, args: &[Object]) -> Result<Object, RuntimeError> {
     if let [loop_] = args {
-        let native = global_module_attr(&ls.globals, "futures", "Future")
+        let native = global_module_attr(&ls.globals, &names().futures, &names().future)
             .is_some_and(|f| crate::stdlib::asyncio_mod::is_future_class(&f));
         // (Not in debug mode: there the Python method's frame shows in
         // the future's `_source_traceback`.)
diff --git a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
index 2d0f4144..2f3bcf5d 100644
--- a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
@@ -510,7 +510,12 @@ fn state_of_obj(obj: &Object) -> Result<Rc<RefCell<FutState>>, RuntimeError> {
 /// or a Python subclass of either).
 fn is_native_future(obj: &Object) -> bool {
     match obj {
-        Object::Instance(inst) => inst.cls().is_subclass_of(&future_class()),
+        Object::Instance(inst) => {
+            if is_exact_native(obj) {
+                return true;
+            }
+            inst.cls().is_subclass_of(future_class_ref())
+        }
         _ => false,
     }
 }
@@ -2193,6 +2198,19 @@ fn task_step_handle_result(
             }
             Ok(())
         }
+        // A bare `yield` (`asyncio.sleep(0)`) reschedules, as the C Task
+        // checks before looking for the future protocol (`None` has no
+        // `_asyncio_future_blocking`, so the order is unobservable, and
+        // the failed lookup built an `AttributeError` each time).
+        SendOutcome::Yield(Object::None) => {
+            call_soon(
+                loop_,
+                task_step_callable(task.clone()),
+                &[],
+                context.clone(),
+            )?;
+            Ok(())
+        }
         SendOutcome::Yield(result) => {
             // The coroutine suspended, yielding `result`.
             let blocking_attr = interp
@@ -2253,15 +2271,6 @@ fn task_step_handle_result(
                     );
                     schedule_step_with_exc(interp, task, loop_, context, new_exc)
                 }
-            } else if matches!(result, Object::None) {
-                // Bare `yield` — reschedule.
-                call_soon(
-                    loop_,
-                    task_step_callable(task.clone()),
-                    &[],
-                    context.clone(),
-                )?;
-                Ok(())
             } else if matches!(result, Object::Generator(_)) {
                 let t = py_repr(interp, task);
                 let r = py_repr(interp, &result);
diff --git a/tests/regrtest/test_asyncio_loop_attr_memo.py b/tests/regrtest/test_asyncio_loop_attr_memo.py
new file mode 100644
index 00000000..02d7a53a
--- /dev/null
+++ b/tests/regrtest/test_asyncio_loop_attr_memo.py
@@ -0,0 +1,126 @@
+"""asyncio's native loop fast paths read the loop's own attributes.
+
+`call_soon`, `create_task` and `create_future` check a stock loop's
+`_debug`, `_closed`, `_ready` and `_task_factory` natively, remembering
+where an instance last held each name. Loops whose attributes sit at
+other positions, debug mode switched on mid-run, and an instance-level
+`get_debug` must all still be seen. A coroutine's bare `yield`
+(`asyncio.sleep(0)`) reschedules its task for the next iteration.
+"""
+
+import asyncio
+import types
+
+
+class OddLoop(asyncio.SelectorEventLoop):
+    """A loop whose instances set extra attributes first, so the names
+    the fast paths read sit at other positions than a stock loop's."""
+
+    def __init__(self):
+        self.a1 = 1
+        self.a2 = 2
+        self.a3 = 3
+        super().__init__()
+
+
+async def ticker(n, log, tag):
+    for i in range(n):
+        log.append((tag, i))
+        await asyncio.sleep(0)
+    return tag
+
+
+async def main(log):
+    a = asyncio.ensure_future(ticker(3, log, "a"))
+    b = asyncio.ensure_future(ticker(3, log, "b"))
+    assert await asyncio.gather(a, b) == ["a", "b"]
+
+
+def run_on(loop):
+    log = []
+    try:
+        loop.run_until_complete(main(log))
+    finally:
+        loop.close()
+    # Bare yields interleave the two tasks one step at a time.
+    assert log == [("a", 0), ("b", 0), ("a", 1), ("b", 1), ("a", 2), ("b", 2)], log
+
+
+# Stock loops and the odd layout, alternately, so each sees the other's
+# remembered positions.
+for _ in range(3):
+    run_on(asyncio.new_event_loop())
+    run_on(OddLoop())
+
+
+# Debug mode switched on while running: handles made afterwards carry a
+# source traceback (the Python `call_soon` path).
+async def flip():
+    loop = asyncio.get_running_loop()
+    h1 = loop.call_soon(lambda: None)
+    assert h1._source_traceback is None
+    loop.set_debug(True)
+    h2 = loop.call_soon(lambda: None)
+    assert h2._source_traceback, h2._source_traceback
+    loop.set_debug(False)
+    h3 = loop.call_soon(lambda: None)
+    assert h3._source_traceback is None
+    await asyncio.sleep(0)
+
+
+for make in (asyncio.new_event_loop, OddLoop):
+    loop = make()
+    try:
+        loop.run_until_complete(flip())
+    finally:
+        loop.close()
+
+
+# An instance-level `get_debug` overrides the stock one.
+loop = OddLoop()
+loop.get_debug = types.MethodType(lambda self: True, loop)
+try:
+    async def check():
+        h = asyncio.get_running_loop().call_soon(lambda: None)
+        assert h._source_traceback, "instance get_debug was ignored"
+
+    loop.run_until_complete(check())
+finally:
+    del loop.get_debug
+    loop.close()
+
+
+# A task factory set on one loop doesn't leak into another.
+made = []
+
+
+def factory(loop, coro, **kw):
+    made.append(coro)
+    return asyncio.Task(coro, loop=loop, **kw)
+
+
+l1, l2 = asyncio.new_event_loop(), OddLoop()
+l1.set_task_factory(factory)
+try:
+    async def noop():
+        return 7
+
+    assert l1.run_until_complete(l1.create_task(noop())) == 7
+    assert l2.run_until_complete(l2.create_task(noop())) == 7
+    assert len(made) == 1, made
+finally:
+    l1.close()
+    l2.close()
+
+
+# A closed loop refuses callbacks.
+loop = OddLoop()
+loop.close()
+try:
+    loop.call_soon(print)
+except RuntimeError as e:
+    assert "closed" in str(e), e
+else:
+    raise AssertionError("call_soon on a closed loop")
+
+print("ok")

From 91a9cd3fee562b34b043332fc0c03d64835a9e1e Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Fri, 9 Oct 2026 23:41:36 -0700
Subject: [PATCH 026/173] perf: clear the globals-__missing__ gate once no
 owner is left

exec, eval and FunctionType register a dict-subclass globals mapping so
global-name misses under it reach its __missing__, and set a gate that
sends every builtin LOAD_GLOBAL (in the frame JIT, the core loop and
leaf plans) down the slow path while any is registered. Pruning dropped
dead owners but never lowered the gate, so one @dataclass whose
annotations needed annotationlib's _StringifierDict (_colorize, hence
traceback, logging and asyncio) slowed builtin lookups for the rest of
the process: a builtin-calling function compiled by the frame JIT left
native code at every such LOAD_GLOBAL (1579 -> 3020 instructions per
call of a two-builtin function after import asyncio).

A prune now sets the gate to whether any owner is left, and runs every
64th finalizer drain while the gate is up (owners die when the code
that used them is done, with nothing else to notice).

logging_mix -0.9%, async_tree -0.6%; suite geomean 0.999x.
---
 crates/weavepy-vm/src/lib.rs                 | 33 +++++++++-
 tests/regrtest/test_globals_missing_prune.py | 68 ++++++++++++++++++++
 2 files changed, 99 insertions(+), 2 deletions(-)
 create mode 100644 tests/regrtest/test_globals_missing_prune.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index bd766105..ca50405c 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -897,9 +897,13 @@ pub struct Interpreter {
     /// native dict alive (CPython's function keeps the mapping itself),
     /// so a weak owner would die with the caller's last direct reference.
     globals_missing_hooks: RefCell<Vec<(usize, Rc<PyInstance>)>>,
-    /// Whether `globals_missing_hooks` has ever been non-empty: the leaf
-    /// paths' one-load gate before the cell borrow.
+    /// Whether `globals_missing_hooks` is non-empty (as of its last
+    /// prune): the leaf paths' one-load gate before the cell borrow. While
+    /// it's set, every builtin `LOAD_GLOBAL` takes the slow path.
     globals_missing_any: Cell<bool>,
+    /// Finalizer drains left before the next prune of
+    /// `globals_missing_hooks` while `globals_missing_any` is set.
+    globals_missing_prune_in: Cell<u32>,
     /// Non-zero while the VM is lazily loading one of its own machinery
     /// modules (e.g. `importlib._bootstrap` for `module.__repr__`). In
     /// CPython those are frozen and fully initialized before user code
@@ -1225,6 +1229,7 @@ impl Default for Interpreter {
             unraisable_hook,
             globals_missing_hooks: RefCell::new(Vec::new()),
             globals_missing_any: Cell::new(false),
+            globals_missing_prune_in: Cell::new(0),
             internal_import_depth: 0,
             frame_locals_pool: ThreadCell::new(Vec::new()),
             inline_pool: Vec::new(),
@@ -1354,6 +1359,7 @@ impl Interpreter {
             unraisable_hook: self.unraisable_hook.clone(),
             globals_missing_hooks: RefCell::new(Vec::new()),
             globals_missing_any: Cell::new(false),
+            globals_missing_prune_in: Cell::new(0),
             internal_import_depth: 0,
             frame_locals_pool: ThreadCell::new(Vec::new()),
             inline_pool: Vec::new(),
@@ -2699,6 +2705,10 @@ impl Interpreter {
     /// `sys.unraisablehook` (the default hook prints `Exception ignored
     /// in: …` to stderr, exactly like CPython) so they don't propagate.
     pub fn run_pending_finalizers(&mut self) -> usize {
+        // (Before the drain: an owner the prune lets go of may queue one.)
+        if self.globals_missing_any.get() {
+            self.prune_globals_missing_hooks_sometimes();
+        }
         // RFC 0077 (WS2): the prompt reaper drains after every node it
         // tears down, and for the overwhelming majority nothing was
         // queued. Answer that without publishing the interpreter pointer
@@ -32233,6 +32243,25 @@ impl Interpreter {
             // the dict, and this table's for the instance.
             Rc::strong_count(d) > 1 || Rc::strong_count(inst) > 1
         });
+        // With no owner left, builtin lookups take their fast paths again
+        // (a `@dataclass` whose annotations needed `annotationlib`'s
+        // `_StringifierDict` set the gate for good otherwise).
+        self.globals_missing_any.set(!hooks.is_empty());
+    }
+
+    /// [`Self::prune_globals_missing_hooks`] every 64th finalizer drain
+    /// while the gate is set: owners die when the `exec`/`eval` or
+    /// function that used them is done, with nothing to notice.
+    #[cold]
+    #[inline(never)]
+    fn prune_globals_missing_hooks_sometimes(&self) {
+        let n = self.globals_missing_prune_in.get();
+        if n > 0 {
+            self.globals_missing_prune_in.set(n - 1);
+            return;
+        }
+        self.globals_missing_prune_in.set(63);
+        self.prune_globals_missing_hooks();
     }
 
     fn load_attr(&mut self, obj: &Object, name: &str) -> Result<Object, RuntimeError> {
diff --git a/tests/regrtest/test_globals_missing_prune.py b/tests/regrtest/test_globals_missing_prune.py
new file mode 100644
index 00000000..4f76cc68
--- /dev/null
+++ b/tests/regrtest/test_globals_missing_prune.py
@@ -0,0 +1,68 @@
+"""A dict subclass's `__missing__` serves global-name misses for code run
+under it as globals, for as long as that code can run.
+
+Builtin lookups skip the check once no such globals are left (the
+owners are pruned when nothing can reach them), so a function made
+under them must keep them alive, and a later one must register again.
+"""
+
+import gc
+
+
+class Fallback(dict):
+    def __missing__(self, key):
+        if key == "magic":
+            return 42
+        if key == "len":
+            # Shadows the builtin: a miss asks the mapping first.
+            return lambda x: 100
+        raise KeyError(key)
+
+
+def make(ns):
+    exec("def f():\n    return magic + len('ab')\n", ns)
+    return ns["f"]
+
+
+f = make(Fallback())
+assert f() == 142
+
+# Unrelated builtin-heavy code in between, and collections, which prune
+# the owners nothing reaches.
+for _ in range(3):
+    gc.collect()
+    total = 0
+    for i in range(2000):
+        total += abs(-i) + len("xy")
+    assert total == sum(range(2000)) + 4000
+
+# `f` still holds its globals: the misses still go to `__missing__`, in
+# whatever tier runs it once it's hot.
+for _ in range(5000):
+    assert f() == 142
+
+# Drop it; a fresh subclass-globals run registers again.
+del f
+gc.collect()
+g = make(Fallback())
+for _ in range(5000):
+    assert g() == 142
+
+# eval with a subclass mapping as globals.
+assert eval("magic * 2", Fallback()) == 84
+
+# A dataclass whose annotations name an undefined type (annotationlib's
+# fake globals) leaves builtin lookups working afterwards.
+from dataclasses import dataclass
+
+
+@dataclass
+class P:
+    x: "Undefined"  # noqa: F821
+    y: int = 0
+
+
+assert P(1).x == 1
+gc.collect()
+assert len([1, 2, 3]) == 3 and abs(-5) == 5
+print("ok")

From 329461de7eafe55e43fe87087bff25b45069f333 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 00:59:45 -0700
Subject: [PATCH 027/173] perf: read asyncio loop attributes and future handles
 without guards

The loop checks behind call_soon, create_task and create_future read
each attribute through a borrowed split-values cell, compared the
remembered name byte by byte, and cloned the value only to test it. They
now peek at the values (falling back to the borrow when the cell is
busy or shared), accept a remembered position whose name text sits at
the same address with the same hash, and test _debug, _closed and the
instance overrides in place. A future's state handle is read from its
slots by peek as well.

async_tree -1.3% (at_tree probe 4.74M -> 4.67M instructions per
iteration); suite geomean 1.585 (base 1.586).
---
 crates/weavepy-vm/src/inst_dict.rs            | 22 ++++++++-
 .../weavepy-vm/src/stdlib/asyncio_events.rs   | 49 +++++++++++++------
 crates/weavepy-vm/src/stdlib/asyncio_mod.rs   |  8 ++-
 3 files changed, 61 insertions(+), 18 deletions(-)

diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index 3b431051..f501881c 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -233,6 +233,9 @@ unsafe impl Send for NameMemo {}
 pub struct PosMemo {
     keys: AtomicUsize,
     pos: AtomicUsize,
+    /// The address of the name's text in the table (a hit whose name
+    /// sits at the same address, with the same hash, needs no compare).
+    text: AtomicUsize,
 }
 
 impl PosMemo {
@@ -240,10 +243,20 @@ impl PosMemo {
         Self {
             keys: AtomicUsize::new(0),
             pos: AtomicUsize::new(0),
+            text: AtomicUsize::new(0),
         }
     }
 }
 
+/// The address of a `str` key's text (0 for any other key).
+#[inline(always)]
+fn key_text(key: &DictKey) -> usize {
+    match &key.0 {
+        Object::Str(s) => s.as_ptr() as usize,
+        _ => 0,
+    }
+}
+
 /// The header of an instance's split values allocation; the values
 /// follow it.
 #[repr(C)]
@@ -666,13 +679,20 @@ impl SplitValues {
             // SAFETY: slot `i` is published (below the values' length,
             // which never exceeds the table's).
             let h = unsafe { *keys.hashes[i].get() };
-            if h == hash && keys.get(i).is_some_and(|key| key_names_str(key, name)) {
+            if h == hash
+                && keys.get(i).is_some_and(|key| {
+                    key_text(key) == memo.text.load(Ordering::Relaxed)
+                        || key_names_str(key, name)
+                })
+            {
                 return self.values().get(i);
             }
         }
         let i = keys.position_hashed(n, name, hash)?;
         memo.keys.store(at, Ordering::Relaxed);
         memo.pos.store(i, Ordering::Relaxed);
+        memo.text
+            .store(keys.get(i).map_or(0, key_text), Ordering::Relaxed);
         self.values().get(i)
     }
 
diff --git a/crates/weavepy-vm/src/stdlib/asyncio_events.rs b/crates/weavepy-vm/src/stdlib/asyncio_events.rs
index b429e8d0..5a5fdd75 100644
--- a/crates/weavepy-vm/src/stdlib/asyncio_events.rs
+++ b/crates/weavepy-vm/src/stdlib/asyncio_events.rs
@@ -251,25 +251,45 @@ fn names() -> &'static Names {
     })
 }
 
-/// Instance attribute `n`, read without running code (a split layout
-/// answers a never-set name from its filter, and a set one from the
-/// position it last had).
-fn inst_attr(inst: &PyInstance, n: &AttrName) -> Option<Object> {
+/// `f` of instance attribute `n`, read without running code (a split
+/// layout answers a never-set name from its filter, and a set one from
+/// the position it last had).
+#[inline]
+fn inst_attr_with<R>(inst: &PyInstance, n: &AttrName, f: impl FnOnce(Option<&Object>) -> R) -> R {
     match inst.dict.published() {
-        Some(d) => d
+        Some(d) => f(d
             .borrow()
             .get(&crate::object::StrKeyHashed {
                 s: n.name,
                 hash: n.hash,
-            })
-            .cloned(),
+            })),
         None => {
+            // SAFETY: a read that runs no code; the view ends with `f`,
+            // which only inspects or clones the value.
+            if let Some(split) = unsafe { inst.dict.split_peek() } {
+                return f(split.get_memo(n.name, n.hash, &n.memo));
+            }
             let split = inst.dict.split_cell().borrow();
-            split.get_memo(n.name, n.hash, &n.memo).cloned()
+            f(split.get_memo(n.name, n.hash, &n.memo))
         }
     }
 }
 
+/// Instance attribute `n` (see [`inst_attr_with`]).
+fn inst_attr(inst: &PyInstance, n: &AttrName) -> Option<Object> {
+    inst_attr_with(inst, n, |v| v.cloned())
+}
+
+/// Whether instance attribute `n` is unset.
+fn inst_attr_unset(inst: &PyInstance, n: &AttrName) -> bool {
+    inst_attr_with(inst, n, |v| v.is_none())
+}
+
+/// Whether instance attribute `n` is `False`.
+fn inst_attr_false(inst: &PyInstance, n: &AttrName) -> bool {
+    inst_attr_with(inst, n, |v| matches!(v, Some(Object::Bool(false))))
+}
+
 /// A class (by address and attribute version, which no other class
 /// state ever reuses) a check below already approved on this thread.
 type Verified = std::cell::Cell<(usize, u64)>;
@@ -398,7 +418,7 @@ pub(crate) fn call_soon(
             ls.call_soon_native
                 .get()
                 .is_some_and(|n| resolves_to(&li.cls(), "call_soon", n))
-        }) && inst_attr(li, &names().call_soon).is_none();
+        }) && inst_attr_unset(li, &names().call_soon);
         if native {
             if let Some(h) = fast_call_soon(ls, loop_, &cb, cb_args, &ctx)? {
                 return Ok(h);
@@ -425,7 +445,7 @@ pub(crate) fn get_debug(loop_: &Object) -> Result<bool, RuntimeError> {
     if let (Some(ls), Object::Instance(li)) = (&ls, loop_) {
         let native = verified(&NATIVE_GET_DEBUG, &li.class.borrow(), || {
             resolves_to(&li.cls(), "get_debug", &ls.get_debug)
-        }) && inst_attr(li, &names().get_debug).is_none();
+        }) && inst_attr_unset(li, &names().get_debug);
         if native {
             if let Some(Object::Bool(b)) = inst_attr(li, &names().debug) {
                 return Ok(b);
@@ -452,12 +472,9 @@ fn plain_loop(ls: &LoopState, loop_: &Object) -> bool {
     });
     let n = names();
     stock
-        && matches!(inst_attr(li, &n.debug), Some(Object::Bool(false)))
-        && matches!(
-            inst_attr(li, &n.closed),
-            Some(Object::Bool(false))
-        )
-        && inst_attr(li, &n.get_debug).is_none()
+        && inst_attr_false(li, &n.debug)
+        && inst_attr_false(li, &n.closed)
+        && inst_attr_unset(li, &n.get_debug)
 }
 
 /// `module.attr`, for the module bound to global `module` in `globals`.
diff --git a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
index 2f3bcf5d..204a9e06 100644
--- a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
@@ -462,10 +462,16 @@ fn handle_key() -> &'static str {
 /// The registry handle stored on `inst`, if a state is attached.
 #[inline]
 fn handle_of(inst: &PyInstance) -> Option<i64> {
-    match inst.slots.try_borrow().ok()?.get_hinted(0, handle_key()) {
+    let read = |slots: &crate::types::SlotStorage| match slots.get_hinted(0, handle_key()) {
         Some(Object::Int(h)) => Some(*h),
         _ => None,
+    };
+    // SAFETY: a read of one slot that runs no code.
+    if let Some(slots) = unsafe { inst.slots.peek() } {
+        return read(slots);
     }
+    let slots = inst.slots.try_borrow().ok()?;
+    read(&slots)
 }
 
 /// The already-attached state cell of a Future/Task instance, without

From 0f09882b69f7327f08308fbcba759086630f502d Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 01:15:21 -0700
Subject: [PATCH 028/173] perf: build iter(obj) generators in place and memo
 chain resends

iter(obj) and await obj over a generator-function __iter__/__await__
staged the receiver in a fresh one-element Vec, copied it into pooled
locals and recycled the Vec. A body without cells of its own now gets
the receiver (and any defaults) written straight into pooled locals, as
core_gen_call does for a call's arguments; other bodies stage it in a
pooled scratch vector.

Collapsing a suspended yield-from chain re-decoded each level's
RESUME/JUMP_BACKWARD/SEND triple; a level at the same code and pc as
the one before it (a recursive generator's) now reuses that level's
SEND.

generators_tree -1.0%, async_tree -0.8%, p_coyield probe -1.4%; suite
geomean 0.998x (0.997x vs base).
---
 crates/weavepy-vm/src/lib.rs                  |  53 +++++-
 .../test_generator_instance_iter_chains.py    | 166 ++++++++++++++++++
 2 files changed, 217 insertions(+), 2 deletions(-)
 create mode 100644 tests/regrtest/test_generator_instance_iter_chains.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ca50405c..ba40f79e 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -20226,6 +20226,25 @@ impl Interpreter {
         }
     }
 
+    /// [`Self::collapse_next`], skipping the instruction checks for a
+    /// frame at the code and pc `memo` holds (and recording them there).
+    #[inline(always)]
+    fn collapse_next_memo<'f>(
+        frame: &'f Frame,
+        memo: &mut (*const CodeObject, u32, usize),
+    ) -> Option<(&'f Rc<PyGenerator>, usize)> {
+        let code = Rc::as_ptr(&frame.code);
+        if code == memo.0 && frame.pc == memo.1 {
+            return match frame.stack.last() {
+                Some(Object::Generator(g) | Object::Coroutine(g)) => Some((g, memo.2)),
+                _ => None,
+            };
+        }
+        let found = Self::collapse_next(frame)?;
+        *memo = (code, frame.pc, found.1);
+        Some(found)
+    }
+
     /// The `SEND` of `None` at `send_pc` of `sw`'s (synced) running
     /// activation to `first` (frame `first_frame`, both checked by
     /// [`Self::core_send_hop`]), a generator itself suspended in a
@@ -20258,6 +20277,9 @@ impl Interpreter {
         // The code objects whose lean resume a level already vouched for
         // (a recursive generator's levels share one).
         let mut code_ok: *const CodeObject = std::ptr::null();
+        // The last level's code, pc and `SEND` (a recursive generator's
+        // levels mostly share all three).
+        let mut resend: (*const CodeObject, u32, usize) = (std::ptr::null(), u32::MAX, 0);
         loop {
             // The level delegates from here (`Delegate::suspend` undoes it).
             // SAFETY: each level is alive (its holder is the level above,
@@ -20282,7 +20304,8 @@ impl Interpreter {
             g = Rc::as_ptr(inner);
             gf = inner_frame;
             // SAFETY: as above.
-            let Some((next, next_send)) = Self::collapse_next(unsafe { &*gf }) else {
+            let Some((next, next_send)) = Self::collapse_next_memo(unsafe { &*gf }, &mut resend)
+            else {
                 break;
             };
             let Some(next_frame) = Self::collapse_frame_with(next, &mut code_ok) else {
@@ -56224,7 +56247,33 @@ impl Interpreter {
         if !code.is_generator {
             return None;
         }
-        self.start_generator_fast(&f, &code, vec![v.clone()])
+        let start = self.generator_start(&f, &code, 1)?;
+        // A body without cells of its own: `self` goes straight into the
+        // locals, as `core_gen_call` passes a call's arguments.
+        if let Some(cells) = f.lean_cells_ref(&code) {
+            let nlocals = code.varnames.len();
+            let locals = self.pooled_locals_empty(nlocals);
+            {
+                // SAFETY: sole owner (see `pooled_locals_empty`).
+                let l = unsafe { &mut *locals.as_ptr() };
+                l.push(v.clone());
+                let missing = (code.arg_count as usize).saturating_sub(1);
+                if missing > 0 {
+                    l.extend(f.defaults[f.defaults.len() - missing..].iter().cloned());
+                }
+                l.resize(nlocals, Object::Unbound);
+            }
+            let mut frame = Self::gen_frame_lean(&f, code.clone(), locals, cells.clone(), start);
+            frame.stack = self.pooled_stack();
+            return Some(Self::wrap_started_generator(
+                &f,
+                &code,
+                self.box_gen_frame(frame),
+            ));
+        }
+        let mut positional = self.pooled_scratch();
+        positional.push(v.clone());
+        Some(self.start_generator_at(&f, &code, positional, start))
     }
 
     /// The generator-family object a call of `f` (its code `code`) with
diff --git a/tests/regrtest/test_generator_instance_iter_chains.py b/tests/regrtest/test_generator_instance_iter_chains.py
new file mode 100644
index 00000000..28bc897a
--- /dev/null
+++ b/tests/regrtest/test_generator_instance_iter_chains.py
@@ -0,0 +1,166 @@
+"""Generators made by `iter(obj)` / `await obj` from a generator
+`__iter__` / `__await__`, and recursive `yield from` chains.
+
+`iter(obj)` builds the generator of a generator-function `__iter__`
+without the call machinery; the receiver goes straight into the frame,
+defaults fill any later parameters, and a body with cells of its own
+takes the general frame. Resuming a suspended chain of `yield from`s
+resumes its innermost generator directly; levels that share code and a
+resume point are walked from what the previous level showed. Values,
+`send`, `throw`, `close`, `gi_yieldfrom` and `gi_running` must come out
+as CPython's.
+"""
+
+import sys
+
+
+class Tree:
+    __slots__ = ("left", "value", "right")
+
+    def __init__(self, left, value, right):
+        self.left = left
+        self.value = value
+        self.right = right
+
+    def __iter__(self):
+        if self.left:
+            yield from self.left
+        yield self.value
+        if self.right:
+            yield from self.right
+
+
+def build(lo, hi):
+    if lo >= hi:
+        return None
+    mid = (lo + hi) // 2
+    return Tree(build(lo, mid), mid, build(mid + 1, hi))
+
+
+tree = build(0, 200)
+for _ in range(30):
+    assert list(tree) == list(range(200))
+    assert sum(tree) == sum(range(200))
+    assert max(x for x in tree if x % 7 == 3) == 199 - (199 - 3) % 7
+
+
+# Defaults after `self`, and a body with a cell of its own.
+class WithDefault:
+    def __iter__(self, step=2, start=1):
+        yield from range(start, 7, step)
+
+
+class WithCell:
+    def __init__(self, n):
+        self.n = n
+
+    def __iter__(self):
+        n = self.n
+
+        def bump(x):
+            return x + n
+
+        for i in range(3):
+            yield bump(i)
+
+
+for _ in range(50):
+    assert list(WithDefault()) == [1, 3, 5]
+    assert list(WithCell(10)) == [10, 11, 12]
+
+
+# `__await__` as a generator function.
+class Ready:
+    def __init__(self, v):
+        self.v = v
+
+    def __await__(self):
+        yield None
+        return self.v
+
+
+async def use(n):
+    total = 0
+    for i in range(n):
+        total += await Ready(i)
+    return total
+
+
+def drive(c):
+    try:
+        while True:
+            c.send(None)
+    except StopIteration as e:
+        return e.value
+
+
+for _ in range(20):
+    assert drive(use(50)) == sum(range(50))
+
+
+# A deep chain: send and throw reach the innermost level; the levels in
+# between report the delegation as CPython does.
+def leaf(log):
+    while True:
+        try:
+            got = yield "leaf"
+            log.append(got)
+        except KeyError as e:
+            log.append(("caught", e.args[0]))
+            yield "after-throw"
+
+
+def level(n, log):
+    if n == 0:
+        r = yield from leaf(log)
+    else:
+        r = yield from level(n - 1, log)
+    return r
+
+
+for depth in (1, 2, 5, 12):
+    log = []
+    g = level(depth, log)
+    assert next(g) == "leaf"
+    for i in range(20):
+        assert g.send(i) == "leaf"
+    assert log == list(range(20)), log
+    # Walk the chain: every level but the innermost delegates.
+    inner, levels = g, 0
+    while inner.gi_yieldfrom is not None:
+        assert not inner.gi_running
+        inner = inner.gi_yieldfrom
+        levels += 1
+    assert levels == depth + 1, (levels, depth)
+    assert inner.gi_code.co_name == "leaf"
+    assert g.throw(KeyError("k")) == "after-throw"
+    assert log[-1] == ("caught", "k")
+    assert next(g) == "leaf"
+    g.close()
+    assert g.gi_frame is None
+
+
+# A level touched from inside the innermost one is running.
+def outer_check():
+    me = yield
+    yield from inner_check(me)
+
+
+def inner_check(top):
+    assert top.gi_running
+    try:
+        next(top)
+    except ValueError as e:
+        assert "already executing" in str(e)
+    else:
+        raise AssertionError("re-entered a running generator")
+    yield "ok"
+
+
+for _ in range(10):
+    g = outer_check()
+    next(g)
+    assert g.send(g) == "ok"
+    g.close()
+
+print("ok")

From dac3140ee2ec60c89d7f1979b487dd822e0e74ee Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 00:27:10 -0700
Subject: [PATCH 029/173] perf: keep the cycle collector's registry in a slab

The collector tracked each object with a separately allocated handle
(an `Arc<TrackedHandle>` of 48 bytes), found it through a hash set
keyed by address, and built a strong candidate copy (48 bytes) plus an
id-to-position hash map of every candidate for each collection, so a
collection paid several hash probes and allocations per object.

The registry is now a slab of 32-byte entries, one list of slots per
generation, and an id index only for the kinds that can't record their
own slot. Instances, classes, functions and the generator family carry
their slot (`GcSlot`, in what was padding for instances), so
registering, finding one from an edge, and an instance's death never
hash. A collection marks the generation lists in place: each entry
holds its candidate flag and `gc_refs`, the walk reads objects through
reference-free views, and only temporaries and the unreachable are
held strongly. The mark walk reads the common containers' cells
without borrow bookkeeping, visits an instance's class without a
clone, offers external traverse hooks only the kinds they match, and
caches index hits for children seen repeatedly (module namespaces).
A tuple of atomic values is no longer promoted to a temporary
candidate (CPython untracks those at creation), which graph_search's
edge lists produced by the hundred thousand.

Instruction counts per iteration (macOS, PYTHONHASHSEED=0):
  gc.collect() over 2N live objects   36.0k -> 25.5k
  self-cyclic instance churn           4.31k -> 3.39k
  N cyclic instances, one collect      5.07k -> 3.96k
  instance holding a list, kept       10.68k -> 8.50k
Suite (ins_suite vs m1): async_tree 0.982, comprehensions 0.965,
xml_etree 0.987, graph_search 0.992, geomean 0.997.
Peak RSS: async_tree 103 -> 77 MB, graph_search 61 -> 47 MB, the
live-collection probe 107 -> 62 MB, the kept-instances probe 80 -> 64 MB.
---
 crates/weavepy-capi/src/gc_bridge.rs          |    9 +-
 crates/weavepy-vm/src/builtins.rs             |    3 +
 crates/weavepy-vm/src/gc_trace.rs             | 1800 ++++++++++-------
 crates/weavepy-vm/src/lib.rs                  |   58 +-
 crates/weavepy-vm/src/object.rs               |   12 +-
 crates/weavepy-vm/src/rc.rs                   |   26 +
 crates/weavepy-vm/src/shared_value.rs         |   13 +
 crates/weavepy-vm/src/stdlib/asyncio_mod.rs   |    5 +-
 crates/weavepy-vm/src/stdlib/gc_real.rs       |   29 +-
 .../src/stdlib/multiprocessing_mod.rs         |    2 +
 crates/weavepy-vm/src/stdlib/thread_real.rs   |    3 +
 crates/weavepy-vm/src/stdlib/weakref_real.rs  |    1 +
 crates/weavepy-vm/src/types.rs                |   65 +-
 crates/weavepy-vm/src/weak_object.rs          |   36 +
 tests/regrtest/test_gc_instance_registry.py   |  147 ++
 15 files changed, 1461 insertions(+), 748 deletions(-)
 create mode 100644 tests/regrtest/test_gc_instance_registry.py

diff --git a/crates/weavepy-capi/src/gc_bridge.rs b/crates/weavepy-capi/src/gc_bridge.rs
index 6aa96bed..3275331f 100644
--- a/crates/weavepy-capi/src/gc_bridge.rs
+++ b/crates/weavepy-capi/src/gc_bridge.rs
@@ -144,8 +144,10 @@ fn has_clear(obj: &Object) -> bool {
 /// once from [`crate::interp::ensure_initialised`], before any extension
 /// code (hence any C GC type) can run.
 pub fn install() {
-    weavepy_vm::gc_trace::register_traverse(has_traverse, traverse);
-    weavepy_vm::gc_trace::register_clear(has_clear, clear);
+    // (Both match instances only; see `instance_type_ptr`.)
+    let kind = weavepy_vm::gc_trace::hook_kind::INSTANCE;
+    weavepy_vm::gc_trace::register_traverse_for(kind, has_traverse, traverse);
+    weavepy_vm::gc_trace::register_clear_for(kind, has_clear, clear);
 }
 
 // ====================================================================
@@ -253,8 +255,7 @@ pub unsafe extern "C" fn PyObject_GC_IsTracked(op: *mut c_void) -> c_int {
         return 0;
     }
     let obj = unsafe { crate::object::clone_object(op as *mut PyObject) };
-    let id = weavepy_vm::weakref_registry::id_of(&obj);
-    c_int::from(weavepy_vm::gc_trace::is_tracked(id))
+    c_int::from(weavepy_vm::gc_trace::is_tracked(&obj))
 }
 
 /// `PyObject_GC_Del(op)` — free a GC object's storage (a GC type's
diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 4e8e733b..e7470fc5 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -8720,6 +8720,7 @@ pub(crate) fn make_unbound_super(class: Rc<crate::types::TypeObject>) -> Object
         finalize_ran: crate::sync::Cell::new(false),
         deferred: crate::sync::Cell::new(false),
         c_body: crate::types::CBody::default(),
+        gc_slot: crate::gc_trace::GcSlot::new(),
     };
     Object::Instance(Rc::new(inst))
 }
@@ -8799,6 +8800,7 @@ pub(crate) fn build_super_proxy(
         finalize_ran: crate::sync::Cell::new(false),
         deferred: crate::sync::Cell::new(false),
         c_body: crate::types::CBody::default(),
+        gc_slot: crate::gc_trace::GcSlot::new(),
     };
     Object::Instance(Rc::new(inst))
 }
@@ -10608,6 +10610,7 @@ fn b_mark_iterable_coroutine(args: &[Object]) -> Result<Object, RuntimeError> {
         closure_cells: std::sync::OnceLock::new(),
         // The copied slot store carries any override along.
         defaults_override: crate::object::OverrideFlag::new(f.defaults_maybe_overridden()),
+        gc_slot: crate::gc_trace::GcSlot::new(),
     };
     Ok(Object::Function(Rc::new(marked)))
 }
diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs
index 7462ce37..935c6009 100644
--- a/crates/weavepy-vm/src/gc_trace.rs
+++ b/crates/weavepy-vm/src/gc_trace.rs
@@ -10,10 +10,7 @@
 //! so objects — and the cycles they form — routinely span OS threads.
 //! A single shared `GcState` is the only design that can break a
 //! cross-thread cycle, and it mirrors CPython's one-collector-per-
-//! interpreter model. `Arc<TrackedHandle>` gives the collector and
-//! the weakref registry shared ownership of each slot; the `Arc` is
-//! genuinely `Send + Sync` now, so no Clippy suppression is needed for
-//! it.
+//! interpreter model.
 //!
 //! The model:
 //!
@@ -73,12 +70,10 @@
 
 use crate::fasthash::ObjectIdHasher;
 use crate::shared_value::ThinArc;
-use crate::sync::Rc as HandleRc;
+
 use crate::sync::RefCell;
 use std::hash::BuildHasherDefault;
-use std::sync::atomic::{
-    AtomicBool, AtomicI64, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering,
-};
+use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize, Ordering};
 
 use crate::object::Object;
 use crate::weak_object::WeakObject;
@@ -87,34 +82,9 @@ use crate::weakref_registry::{id_of, ObjectId};
 /// A set of object ids (addresses), hashed with the address mixer.
 type IdSet = std::collections::HashSet<ObjectId, BuildHasherDefault<ObjectIdHasher>>;
 
-/// The collector's id index: each tracked object's handle, found by the
-/// id the handle carries (one word a bucket, against two for a map from
-/// the id to the handle).
-type GcIndex = std::collections::HashSet<IndexEntry, BuildHasherDefault<ObjectIdHasher>>;
-
-/// A [`GcIndex`] entry: a handle, hashed, compared and looked up by its id.
-struct IndexEntry(HandleRc<TrackedHandle>);
-
-impl std::hash::Hash for IndexEntry {
-    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-        // As `ObjectId` hashes, so a lookup by id finds it.
-        self.0.id.hash(state);
-    }
-}
-
-impl PartialEq for IndexEntry {
-    fn eq(&self, other: &Self) -> bool {
-        self.0.id == other.0.id
-    }
-}
-
-impl Eq for IndexEntry {}
-
-impl std::borrow::Borrow<ObjectId> for IndexEntry {
-    fn borrow(&self) -> &ObjectId {
-        &self.0.id
-    }
-}
+/// The registry's id index: a tracked object's slot by its payload
+/// address, for the kinds that don't record their slot themselves.
+type GcIndex = std::collections::HashMap<ObjectId, u32, BuildHasherDefault<ObjectIdHasher>>;
 
 /// The standard CPython generation count (3) and default
 /// thresholds (CPython 3.14's): gen 0 collects when 2000 net tracked allocations
@@ -123,7 +93,6 @@ impl std::borrow::Borrow<ObjectId> for IndexEntry {
 pub const N_GENERATIONS: usize = 3;
 // Color and generation are bounded states; reference counts remain full-width.
 const _: () = assert!(N_GENERATIONS > 0 && N_GENERATIONS <= u8::MAX as usize + 1);
-const MAX_GENERATION: u8 = (N_GENERATIONS - 1) as u8;
 pub const DEFAULT_THRESHOLDS: [usize; N_GENERATIONS] = [2000, 10, 10];
 
 /// Upper bound on the number of mark-sweep passes a single
@@ -155,156 +124,297 @@ pub trait Finalize {
     fn finalize(&self);
 }
 
-/// A tracked object's entry: a non-owning handle on the object plus the
-/// bookkeeping that outlives a collection. The collector never keeps an
-/// object alive. An entry whose object died stays in its generation until
-/// the next collection or prune of that generation drops it; its weak
-/// handle keeps the allocation reserved until then, so the address the
-/// index keys it by can't be reused by another object meanwhile.
-#[allow(missing_debug_implementations)]
-pub struct TrackedHandle {
-    /// The tracked object, held weakly.
-    pub object: WeakObject,
-    /// Identity, computed from `id_of(object)` at `track` time.
-    pub id: ObjectId,
-    /// [`color::Frozen`] while `gc.freeze()` holds the entry, otherwise
-    /// [`color::White`]. Collections keep their marking state in the
-    /// candidates they build, not here.
-    pub color: AtomicU8,
-    /// Generation index (0..N_GENERATIONS). Survivors are promoted by
-    /// incrementing this.
-    pub generation: AtomicU8,
-    /// Position of this handle within its owning `Vec` —
-    /// `generations[generation].handles` normally, or the `frozen`
-    /// list when `color == Frozen`. Maintained by every site that
-    /// pushes, drains, or rebuilds those vectors so that
-    /// [`GcState::untrack_id`] can `swap_remove` in O(1).
-    pub slot: CachedSlot,
-    /// Has this object's `__del__` already *run* to completion? CPython
-    /// guarantees a finaliser runs at most once.
-    pub finalized: AtomicBool,
-    /// Has this object's `__del__` been *queued* by a collection but not yet
-    /// run? While set, the object is excluded from the `collected` count:
-    /// its finalizer (drained after `gc.collect()` returns) may resurrect
-    /// it, and CPython only counts objects that are actually reclaimed.
-    pub finalize_queued: AtomicBool,
-}
-
-/// A compact vector-position hint, with separate absent and uncached states.
-///
-/// Positions too large to cache use the pointer-search fallback. Neither
-/// decoded sentinel can index a live `Vec<Arc<TrackedHandle>>`, whose
-/// allocation is bounded by `isize::MAX`.
+/// No registry slot: the [`GcSlot`] of an object the collector doesn't
+/// hold, and the end of the free list.
+pub const NO_SLOT: u32 = u32::MAX;
+
+/// The registry slot an object of a kind that records its own (see
+/// [`Registry`]) carries: [`NO_SLOT`] while it isn't registered. A copy of
+/// the object is another object, so a clone starts unregistered.
 #[derive(Debug)]
-pub struct CachedSlot(AtomicU32);
-
-impl CachedSlot {
-    const fn encode(slot: usize) -> u32 {
-        if slot == usize::MAX {
-            u32::MAX
-        } else if slot >= (u32::MAX - 1) as usize {
-            u32::MAX - 1
-        } else {
-            slot as u32
-        }
+pub struct GcSlot(crate::sync::Cell<u32>);
+
+impl GcSlot {
+    pub fn new() -> Self {
+        Self(crate::sync::Cell::new(NO_SLOT))
     }
 
-    const fn decode(slot: u32) -> usize {
-        if slot == u32::MAX {
-            usize::MAX
-        } else if slot == u32::MAX - 1 {
-            usize::MAX - 1
-        } else {
-            slot as usize
-        }
+    #[inline]
+    pub fn get(&self) -> u32 {
+        self.0.get()
     }
 
-    pub const fn new(slot: usize) -> Self {
-        Self(AtomicU32::new(Self::encode(slot)))
+    #[inline]
+    fn set(&self, slot: u32) {
+        self.0.set(slot);
     }
 
+    /// Whether the object has a registry entry.
     #[inline]
-    pub fn load(&self, order: Ordering) -> usize {
-        Self::decode(self.0.load(order))
+    pub fn is_registered(&self) -> bool {
+        self.get() != NO_SLOT
+    }
+}
+
+impl Default for GcSlot {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Clone for GcSlot {
+    fn clone(&self) -> Self {
+        Self::new()
     }
+}
 
+/// [`Entry::gen`] of an entry `gc.freeze()` holds (it sits in the frozen
+/// list).
+const GEN_FROZEN: u8 = 0xfe;
+/// [`Entry::gen`] of a free slot.
+const GEN_FREE: u8 = 0xff;
+
+/// [`Entry::flags`]: the object's `__del__` has run to completion
+/// (CPython guarantees a finalizer runs at most once).
+const F_FINALIZED: u8 = 1;
+/// The object's `__del__` was queued by a collection and hasn't run yet.
+/// While set, the object is excluded from the `collected` count: its
+/// finalizer (drained after `gc.collect()` returns) may resurrect it, and
+/// CPython only counts objects that are actually reclaimed.
+const F_QUEUED: u8 = 2;
+/// The object is a candidate of the collection in progress; the entry's
+/// `gc_refs` holds its marking count.
+const F_CAND: u8 = 4;
+/// The collection in progress found the candidate reachable from outside
+/// the candidates (its marking color is grey or black).
+const F_REACHED: u8 = 8;
+
+/// One registered object: a non-owning handle on it plus the bookkeeping
+/// that outlives a collection. The collector never keeps an object alive.
+/// An entry whose object died stays in its generation until its removal
+/// (an instance removes its own from its `Drop`) or the next prune or
+/// collection of that generation; its weak handle keeps the allocation
+/// reserved until then, so the address the index keys it by can't be
+/// reused by another object meanwhile.
+struct Entry {
+    /// The object, held weakly; `None` for a free slot.
+    obj: Option<WeakObject>,
+    /// Position in the owning list (`gens[gen]`, or `frozen`), or the next
+    /// free slot for a free one.
+    pos: u32,
+    /// While [`F_CAND`] is set: the object's references from outside the
+    /// candidates, as the collection counts them (CPython's `gc_refs`).
+    gc_refs: std::cell::Cell<i32>,
+    /// Generation (0..N_GENERATIONS), [`GEN_FROZEN`] or [`GEN_FREE`].
+    gen: u8,
+    /// [`F_FINALIZED`], [`F_QUEUED`] and [`F_CAND`].
+    flags: std::cell::Cell<u8>,
+}
+
+impl Entry {
     #[inline]
-    pub fn store(&self, slot: usize, order: Ordering) {
-        self.0.store(Self::encode(slot), order);
+    fn has(&self, flag: u8) -> bool {
+        self.flags.get() & flag != 0
     }
 
     #[inline]
-    pub fn swap(&self, slot: usize, order: Ordering) -> usize {
-        Self::decode(self.0.swap(Self::encode(slot), order))
-    }
-}
-
-#[allow(non_upper_case_globals)]
-pub mod color {
-    pub const White: u8 = 0;
-    pub const Grey: u8 = 1;
-    pub const Black: u8 = 2;
-    pub const Frozen: u8 = 3;
-}
-
-impl TrackedHandle {
-    /// A generation-`generation` entry for `object`, or `None` for a value
-    /// with no heap allocation of its own.
-    pub fn new(object: &Object, generation: usize) -> Option<Self> {
-        assert!(generation < N_GENERATIONS, "invalid collector generation");
-        Some(Self {
-            id: id_of(object),
-            object: WeakObject::new(object)?,
-            color: AtomicU8::new(color::White),
-            generation: AtomicU8::new(generation as u8),
-            slot: CachedSlot::new(0),
-            finalized: AtomicBool::new(false),
-            finalize_queued: AtomicBool::new(false),
-        })
+    fn set(&self, flag: u8, on: bool) {
+        let f = self.flags.get();
+        self.flags.set(if on { f | flag } else { f & !flag });
     }
 }
 
-/// Swap-remove the handle at `slot` from `vec`, fixing up the slot
-/// index of whatever handle gets moved into the vacated position.
-/// O(1): the only handle whose position changes is the one swapped
-/// in from the end, and its `slot` field is corrected here so the
-/// per-handle position invariant holds after the call.
+/// Whether the registry finds `w`'s object through the object's own
+/// [`GcSlot`] (instances, classes, functions and the generator family)
+/// rather than the id index.
 #[inline]
-fn swap_remove_handle(vec: &mut Vec<HandleRc<TrackedHandle>>, slot: usize) {
-    if slot >= vec.len() {
-        return;
-    }
-    vec.swap_remove(slot);
-    if let Some(moved) = vec.get(slot) {
-        moved.slot.store(slot, Ordering::Release);
+fn slot_is_intrusive(w: &WeakObject) -> bool {
+    matches!(
+        w,
+        WeakObject::Instance(_)
+            | WeakObject::Type(_)
+            | WeakObject::Function(_)
+            | WeakObject::Generator(_)
+            | WeakObject::Coroutine(_)
+            | WeakObject::AsyncGenerator(_)
+    )
+}
+
+/// The [`GcSlot`] of a kind that records its own registry slot.
+#[inline]
+fn intrusive_slot(obj: &Object) -> Option<&GcSlot> {
+    match obj {
+        Object::Instance(i) => Some(&i.gc_slot),
+        Object::Type(t) => Some(&t.gc_slot),
+        Object::Function(f) => Some(&f.gc_slot),
+        Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => Some(&g.gc_slot),
+        _ => None,
     }
 }
 
-/// Correctness fallback for [`GcState::untrack_id`]: when a handle's cached
-/// `slot` no longer points at it (a concurrent `swap_remove`/promotion on
-/// another OS thread moved it before this thread acquired the vector lock),
-/// locate it by pointer identity and `swap_remove` it. Returns `true` if the
-/// handle was found and removed. O(n) in the generation length, but only ever
-/// taken on the rare stale-cache path — the common case stays O(1).
+/// Forget the registry slot a live object records (its entry is going).
+/// A dead one's field is gone with it.
 #[inline]
-fn remove_handle_by_ptr(
-    vec: &mut Vec<HandleRc<TrackedHandle>>,
-    handle: &HandleRc<TrackedHandle>,
-) -> bool {
-    if let Some(pos) = vec.iter().position(|h| HandleRc::ptr_eq(h, handle)) {
-        swap_remove_handle(vec, pos);
-        true
-    } else {
-        false
+fn clear_intrusive_slot(w: &WeakObject) {
+    fn clear<T: 'static>(w: &crate::sync::Weak<T>, slot: impl Fn(&T) -> &GcSlot) {
+        if w.strong_count() > 0 {
+            // SAFETY: the object is alive (the handle keeps the
+            // allocation, the count says the payload is live), and the
+            // field is an atomic cell.
+            slot(unsafe { &*w.as_ptr() }).set(NO_SLOT);
+        }
+    }
+    match w {
+        WeakObject::Instance(w) => clear(w, |i| &i.gc_slot),
+        WeakObject::Type(w) => clear(w, |t| &t.gc_slot),
+        WeakObject::Function(w) => clear(w, |f| &f.gc_slot),
+        WeakObject::Generator(w) | WeakObject::Coroutine(w) | WeakObject::AsyncGenerator(w) => {
+            clear(w, |g| &g.gc_slot);
+        }
+        _ => {}
     }
 }
 
-#[derive(Default)]
-struct Generation {
-    /// All tracked handles in this generation. Append-only
-    /// during normal allocation; rewritten in place when
-    /// objects are promoted or moved to the unreachable list.
-    handles: Vec<HandleRc<TrackedHandle>>,
+/// The registry of tracked objects: a slab of entries addressed by slot,
+/// one list of slots per generation (and one for the frozen set), and an
+/// id index for the kinds that don't record their own slot.
+///
+/// Every list keeps each member's position in its entry, so a removal is
+/// a swap with the list's last member, and a collection walks a
+/// generation without hashing. An instance records its slot itself
+/// ([`crate::types::PyInstance::gc_slot`]), so registering one, finding
+/// it from an edge, and its death's removal never touch the index.
+struct Registry {
+    slab: Vec<Entry>,
+    /// Head of the free-slot list (threaded through `Entry::pos`).
+    free: u32,
+    gens: [Vec<u32>; N_GENERATIONS],
+    frozen: Vec<u32>,
+    /// Payload address -> slot, for the kinds without an intrusive slot.
+    index: GcIndex,
+}
+
+impl Registry {
+    fn new() -> Self {
+        Self {
+            slab: Vec::new(),
+            free: NO_SLOT,
+            gens: Default::default(),
+            frozen: Vec::new(),
+            index: GcIndex::default(),
+        }
+    }
+
+    #[inline]
+    fn entry(&self, slot: u32) -> &Entry {
+        &self.slab[slot as usize]
+    }
+
+    #[inline]
+    fn list_mut(&mut self, gen: u8) -> &mut Vec<u32> {
+        if gen == GEN_FROZEN {
+            &mut self.frozen
+        } else {
+            &mut self.gens[usize::from(gen)]
+        }
+    }
+
+    /// The slot of the registered object `obj`, if any. `filter` is the
+    /// index's miss-filter.
+    #[inline]
+    fn slot_of(&self, obj: &Object, filter: &crate::hot_filter::RebuildableBloom) -> Option<u32> {
+        if let Some(cell) = intrusive_slot(obj) {
+            let s = cell.get();
+            return (s != NO_SLOT).then_some(s);
+        }
+        let key = obj.payload_addr()? as ObjectId;
+        if !filter.may_contain(key) {
+            return None;
+        }
+        self.index.get(&key).copied()
+    }
+
+    /// Add `obj` to list `gen` in a fresh slot.
+    fn insert(&mut self, obj: WeakObject, gen: u8, flags: u8) -> u32 {
+        let pos = self.list_mut(gen).len() as u32;
+        let entry = Entry {
+            obj: Some(obj),
+            pos,
+            gc_refs: std::cell::Cell::new(0),
+            gen,
+            flags: std::cell::Cell::new(flags),
+        };
+        let slot = if self.free != NO_SLOT {
+            let slot = self.free;
+            let e = &mut self.slab[slot as usize];
+            self.free = e.pos;
+            *e = entry;
+            slot
+        } else {
+            let slot = u32::try_from(self.slab.len()).expect("collector registry overflow");
+            // (A collection's grey stack tags temporaries with the top bit.)
+            assert!(slot < TEMP_BIT, "collector registry overflow");
+            self.slab.push(entry);
+            slot
+        };
+        self.list_mut(gen).push(slot);
+        slot
+    }
+
+    /// Take `slot` out of its list (a swap with the list's last member).
+    fn unlink(&mut self, slot: u32) {
+        let (gen, pos) = {
+            let e = self.entry(slot);
+            (e.gen, e.pos as usize)
+        };
+        let list = self.list_mut(gen);
+        debug_assert_eq!(list.get(pos), Some(&slot));
+        list.swap_remove(pos);
+        if let Some(&moved) = list.get(pos) {
+            self.slab[moved as usize].pos = pos as u32;
+        }
+    }
+
+    /// Return an unlinked slot to the free list, handing back its handle
+    /// (whose drop only releases memory).
+    fn release(&mut self, slot: u32) -> Option<WeakObject> {
+        let free = self.free;
+        let e = &mut self.slab[slot as usize];
+        let w = e.obj.take();
+        e.gen = GEN_FREE;
+        e.flags.set(0);
+        e.pos = free;
+        self.free = slot;
+        w
+    }
+
+    /// Drop `slot`'s entry altogether: from its list, the index, and a live
+    /// instance's own record.
+    fn remove(&mut self, slot: u32) -> Option<WeakObject> {
+        self.unlink(slot);
+        self.forget_key(slot);
+        self.release(slot)
+    }
+
+    /// Drop the index entry (or a live instance's record) naming `slot`.
+    fn forget_key(&mut self, slot: u32) {
+        let Some(w) = self.slab[slot as usize].obj.as_ref() else {
+            return;
+        };
+        if slot_is_intrusive(w) {
+            clear_intrusive_slot(w);
+        } else {
+            let key = w.addr() as ObjectId;
+            if self.index.get(&key) == Some(&slot) {
+                self.index.remove(&key);
+            }
+        }
+    }
+
+    /// Every live member of every list, frozen ones included.
+    fn len(&self) -> usize {
+        self.gens.iter().map(Vec::len).sum::<usize>() + self.frozen.len()
+    }
 }
 
 #[derive(Debug, Default, Clone, Copy)]
@@ -314,40 +424,26 @@ pub struct GcStats {
     pub uncollectable: u64,
 }
 
-/// One object under examination by a collection: a strong reference for
-/// the collection's duration, its entry (absent for a temporary candidate
-/// discovered through an untracked container), and the marking state.
-struct Cand {
+/// The top bit of a grey-stack item: the rest is a temporary candidate's
+/// position rather than a registry slot.
+const TEMP_BIT: u32 = 1 << 31;
+
+/// A temporary candidate: an untracked object a collection found a cycle
+/// could route through (see `collect_generation`'s phase 2), held for the
+/// collection's duration, with its marking state.
+struct Temp {
     obj: Object,
-    id: ObjectId,
-    entry: Option<HandleRc<TrackedHandle>>,
     gc_refs: std::cell::Cell<i64>,
-    color: std::cell::Cell<u8>,
+    reached: std::cell::Cell<bool>,
 }
 
-impl Cand {
-    fn new(obj: Object, id: ObjectId, entry: Option<HandleRc<TrackedHandle>>) -> Self {
-        Self {
-            obj,
-            id,
-            entry,
-            gc_refs: std::cell::Cell::new(0),
-            color: std::cell::Cell::new(color::White),
-        }
-    }
-
-    fn is_white(&self) -> bool {
-        self.color.get() == color::White
-    }
-
-    /// Whether the object still owes a `__del__` (or a generator close).
-    fn pending_finalizer(&self) -> bool {
-        has_finalizer(&self.obj)
-            && !self
-                .entry
-                .as_ref()
-                .is_some_and(|e| e.finalized.load(Ordering::Acquire))
-    }
+/// A collection's view of one candidate (see [`Finder::find`]).
+#[derive(Clone, Copy)]
+enum Hit {
+    /// A registered candidate, by slot.
+    Real(u32),
+    /// A temporary, by position.
+    Temp(u32),
 }
 
 /// How many entries of the older generations each young collection or
@@ -372,15 +468,13 @@ const OLD_PRUNE_BUDGET: usize = 512;
 /// have to find and break the cycles.
 #[allow(missing_debug_implementations)]
 pub struct GcState {
-    generations: RefCell<[Generation; N_GENERATIONS]>,
-    /// Insert-only miss-filter over `index`, maintained at
-    /// [`Self::track_now`] and consulted by the usually-miss `is_tracked`
-    /// probes. Rebuilt from `index` once most of its bits name objects
-    /// long gone.
+    /// Every tracked object, by generation (see [`Registry`]).
+    reg: RefCell<Registry>,
+    /// Insert-only miss-filter over the registry's id index, maintained at
+    /// [`Self::register`] and consulted by the usually-miss lookups.
+    /// Rebuilt from the index once most of its bits name objects long
+    /// gone.
     tracked_filter: crate::hot_filter::RebuildableBloom,
-    /// Id → handle index over every tracked object (all generations plus
-    /// the frozen set): keeps `track` dedupe and `is_tracked` O(1).
-    index: RefCell<GcIndex>,
     /// Re-entrancy guard: a nested collection would see torn generation
     /// lists. Atomic so the whole `GcState` is `Sync`.
     collecting: AtomicBool,
@@ -441,10 +535,6 @@ pub struct GcState {
     /// and most die before any collection.
     young_lists: RefCell<Vec<crate::sync::Weak<RefCell<Vec<Object>>>>>,
     young_dicts: RefCell<Vec<crate::sync::Weak<RefCell<crate::object::DictData>>>>,
-    /// Frozen handles. `gc.freeze()` moves all tracked objects
-    /// here; they are skipped by future collections until
-    /// `gc.unfreeze()` runs.
-    frozen: RefCell<Vec<HandleRc<TrackedHandle>>>,
     /// `gc.garbage` — uncollectable objects (cycles whose
     /// finalisers refused to release).
     pub garbage: RefCell<Vec<Object>>,
@@ -479,9 +569,8 @@ impl Default for GcState {
 impl GcState {
     pub fn new() -> Self {
         Self {
-            generations: RefCell::new(Default::default()),
+            reg: RefCell::new(Registry::new()),
             tracked_filter: crate::hot_filter::RebuildableBloom::new(),
-            index: RefCell::new(GcIndex::default()),
             collecting: AtomicBool::new(false),
             thresholds: RefCell::new(DEFAULT_THRESHOLDS),
             counts: RefCell::new([0; N_GENERATIONS]),
@@ -496,7 +585,6 @@ impl GcState {
             young_gens: RefCell::new(Vec::new()),
             young_lists: RefCell::new(Vec::new()),
             young_dicts: RefCell::new(Vec::new()),
-            frozen: RefCell::new(Vec::new()),
             garbage: RefCell::new(Vec::new()),
             callbacks: RefCell::new(Vec::new()),
             stats: RefCell::new([GcStats::default(); N_GENERATIONS]),
@@ -527,8 +615,7 @@ impl GcState {
     /// holds) are consistent.
     pub unsafe fn reinit_after_fork_in_child(this: *mut Self) {
         unsafe {
-            RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).generations));
-            RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).index));
+            RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).reg));
             RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).thresholds));
             RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).counts));
             RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).deferred));
@@ -538,7 +625,6 @@ impl GcState {
             RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).young_gens));
             RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).young_lists));
             RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).young_dicts));
-            RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).frozen));
             RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).garbage));
             RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).callbacks));
             RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).stats));
@@ -566,14 +652,27 @@ impl GcState {
     /// `__del__` returns, so the next collection treats a non-resurrected
     /// object as plain dead garbage (and a resurrected one is never
     /// re-finalized).
-    pub fn complete_finalizer(&self, id: ObjectId) {
-        self.note_finalized(id);
-        if let Some(h) = self.handle_for(id) {
-            h.finalized.store(true, Ordering::Release);
-            h.finalize_queued.store(false, Ordering::Release);
+    pub fn complete_finalizer(&self, obj: &Object) {
+        self.note_finalized(id_of(obj));
+        let reg = self.reg.borrow();
+        if let Some(slot) = reg.slot_of(obj, &self.tracked_filter) {
+            let e = reg.entry(slot);
+            e.set(F_FINALIZED, true);
+            e.set(F_QUEUED, false);
         }
     }
 
+    /// Claim `obj`'s finalizer for the caller: mark it run, returning
+    /// whether it had been claimed before (`None`: `obj` isn't tracked).
+    pub fn claim_finalizer(&self, obj: &Object) -> Option<bool> {
+        let reg = self.reg.borrow();
+        let slot = reg.slot_of(obj, &self.tracked_filter)?;
+        let e = reg.entry(slot);
+        let was = e.has(F_FINALIZED);
+        e.set(F_FINALIZED, true);
+        Some(was)
+    }
+
     /// Track `obj` for cycle detection. Idempotent — if `obj`
     /// is already tracked, this is a no-op.
     pub fn track(&self, obj: &Object) {
@@ -882,114 +981,117 @@ impl GcState {
         }
     }
 
-    /// Enter `obj` in the index and generation 0 (`false`: it was already
+    /// Enter `obj` in the registry's generation 0 (`false`: it was already
     /// there, or has no heap allocation of its own). A `fresh` object's id
     /// may be a finalized one's, recycled; a young instance's own
     /// finalizer may already have run, and its entry says so.
     fn register(&self, obj: &Object, fresh: bool) -> bool {
-        let new_id = id_of(obj);
+        let Some(key) = obj.payload_addr() else {
+            return false;
+        };
+        let key = key as ObjectId;
         {
-            let mut index = self.index.borrow_mut();
+            let mut reg = self.reg.borrow_mut();
             // An entry whose object died can't sit at this address: its
             // weak handle keeps the allocation reserved until the entry is
             // pruned.
-            if index.contains(&new_id) {
+            if reg.slot_of(obj, &self.tracked_filter).is_some() {
                 return false;
             }
-            let Some(handle) = TrackedHandle::new(obj, 0) else {
+            let Some(weak) = WeakObject::new(obj) else {
                 return false;
             };
-            let handle = HandleRc::new(handle);
-            // A generator remembers it has an index entry, which its
-            // finishing release then removes (see `release_finished_gen`).
-            if let Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) = obj {
-                g.gc_registered.set(true);
-            }
-            // Publish to the miss-filter *before* the insert becomes
-            // observable (we hold the index borrow).
-            self.tracked_filter.insert(new_id);
-            index.insert(IndexEntry(handle.clone()));
             // `finalized_ids` is keyed by object id (a pointer), which the
             // allocator recycles. A freshly tracked object at a recycled
             // address must start *un*-finalized (`test_is_finalized`).
             // Almost always empty (only `__del__`-bearing objects ever land
             // there).
-            if !self.finalized_ids.borrow().is_empty() {
+            let mut flags = 0;
+            // SAFETY: a read that runs no code.
+            if unsafe { self.finalized_ids.peek() }.is_none_or(|f| !f.is_empty()) {
+                let id = id_of(obj);
                 if fresh {
-                    self.finalized_ids.borrow_mut().remove(&new_id);
-                } else if self.finalized_ids.borrow().contains(&new_id) {
-                    handle.finalized.store(true, Ordering::Release);
+                    self.finalized_ids.borrow_mut().remove(&id);
+                } else if self.finalized_ids.borrow().contains(&id) {
+                    flags = F_FINALIZED;
+                }
+            }
+            if !slot_is_intrusive(&weak) {
+                // Publish to the miss-filter *before* the insert becomes
+                // observable (we hold the registry borrow).
+                self.tracked_filter.insert(key);
+            }
+            let slot = reg.insert(weak, 0, flags);
+            match intrusive_slot(obj) {
+                Some(cell) => cell.set(slot),
+                None => {
+                    reg.index.insert(key, slot);
+                }
+            }
+            // (Registered other than through `track`, which retires the
+            // deferral itself.)
+            if let Object::Instance(i) = obj {
+                if i.is_gc_deferred() {
+                    i.clear_deferred_tracking();
                 }
             }
-            let mut gens = self.generations.borrow_mut();
-            handle.slot.store(gens[0].handles.len(), Ordering::Release);
-            gens[0].handles.push(handle);
         }
         serial_add(&self.tracked_count, 1);
         serial_add(&self.tracked_version, 1);
         true
     }
 
-    /// Drop the registry entry of a registered object that just died, if
-    /// the registry is free to change: its weak handle otherwise keeps the
-    /// object's allocation alive until a collection of its generation
-    /// prunes it (CPython unlinks a dying object from its GC list at
-    /// once). A busy registry (a collection is releasing garbage) is left
-    /// as it is; that collection prunes the entry.
-    fn forget_dead(&self, id: ObjectId) {
-        let (Ok(mut index), Ok(mut counts), Ok(mut gens)) = (
-            self.index.try_borrow_mut(),
-            self.counts.try_borrow_mut(),
-            self.generations.try_borrow_mut(),
-        ) else {
+    /// Drop the registry entry of an instance that is dying (from its
+    /// `Drop`, with its `gc_slot` and address), if the registry is free to
+    /// change: its weak handle otherwise keeps the object's allocation
+    /// alive until a collection of its generation prunes it (CPython
+    /// unlinks a dying object from its GC list at once). A busy registry
+    /// (a collection is releasing garbage) is left as it is; that
+    /// collection prunes the entry.
+    fn forget_instance(&self, slot: u32, addr: usize) {
+        let (Ok(mut reg), Ok(mut counts)) =
+            (self.reg.try_borrow_mut(), self.counts.try_borrow_mut())
+        else {
             return;
         };
-        let Some(IndexEntry(handle)) = index.get(&id) else {
+        let Some(e) = reg.slab.get(slot as usize) else {
             return;
         };
-        if !handle.object.is_dead() || handle.color.load(Ordering::Acquire) == color::Frozen {
-            return;
+        match &e.obj {
+            Some(WeakObject::Instance(w)) if w.as_ptr() as usize == addr => {}
+            _ => return,
         }
-        let IndexEntry(handle) = index.take(&id).expect("probed above");
-        let g = usize::from(
-            handle
-                .generation
-                .load(Ordering::Acquire)
-                .min(MAX_GENERATION),
-        );
-        let slot = handle.slot.load(Ordering::Acquire);
-        let removed = if gens[g]
-            .handles
-            .get(slot)
-            .is_some_and(|h| HandleRc::ptr_eq(h, &handle))
-        {
-            swap_remove_handle(&mut gens[g].handles, slot);
-            true
-        } else {
-            remove_handle_by_ptr(&mut gens[g].handles, &handle)
-        };
-        if !removed {
-            // Somewhere unexpected: leave it to the collector.
-            index.insert(IndexEntry(handle));
+        if e.gen == GEN_FROZEN {
             return;
         }
-        // (As `untrack_id_in` accounts it.)
+        let weak = reg.remove(slot);
+        // (As `untrack_slot` accounts it.)
         counts[0] = counts[0].saturating_sub(1);
         self.sync_gen0_gauge(counts[0], None);
-        drop((index, counts, gens));
+        drop((reg, counts));
         serial_add(&self.tracked_count, usize::MAX);
         serial_add(&self.tracked_version, 1);
-        drop(handle);
+        drop(weak);
     }
 
     /// Stop tracking `obj`. Backs the explicit `gc._untrack(obj)` extension
     /// and the C-API `PyObject_GC_UnTrack`.
+    pub fn untrack(&self, obj: &Object) {
+        let slot = self.reg.borrow().slot_of(obj, &self.tracked_filter);
+        match slot {
+            Some(slot) => self.untrack_slot(slot, id_of(obj), true),
+            None => self.untrack_young(id_of(obj)),
+        }
+    }
+
+    /// [`Self::untrack`] by id, for an object that isn't an instance.
     pub fn untrack_id(&self, id: ObjectId) {
         self.untrack_id_in(id, true);
     }
 
-    /// [`Self::untrack_id`]; with `young` false, an object in a young set
-    /// is left there (see [`untrack_registered_id`]).
+    /// Stop tracking the object of id `id`, which isn't an instance;
+    /// with `young` false, an object in a young set is left there (see
+    /// [`untrack_registered_id`]).
     fn untrack_id_in(&self, id: ObjectId, young: bool) {
         // A registered object is never in a young set, so the index (a
         // hash probe) answers first; the young sets are scanned only for
@@ -998,14 +1100,28 @@ impl GcState {
         // `YOUNG_CAP` entries.) An index entry left by a dead object
         // whose address a young one now reuses is dropped, and the young
         // sets still searched.
-        let indexed = self.index.borrow_mut().take(&id);
-        let Some(IndexEntry(handle)) = indexed else {
-            if young {
-                self.untrack_young(id);
-            }
-            return;
+        let slot = if self.tracked_filter.may_contain(id) {
+            self.reg.borrow().index.get(&id).copied()
+        } else {
+            None
+        };
+        match slot {
+            Some(slot) => self.untrack_slot(slot, id, young),
+            None if young => self.untrack_young(id),
+            None => {}
+        }
+    }
+
+    /// Remove the entry at `slot` (an object of id `id`), accounting it as
+    /// a deallocation; with `young`, a dead entry's id is also searched in
+    /// the young sets.
+    fn untrack_slot(&self, slot: u32, id: ObjectId, young: bool) {
+        let (weak, dead) = {
+            let mut reg = self.reg.borrow_mut();
+            let dead = reg.entry(slot).obj.as_ref().is_none_or(WeakObject::is_dead);
+            (reg.remove(slot), dead)
         };
-        if young && handle.object.is_dead() {
+        if young && dead {
             self.untrack_young(id);
         }
         {
@@ -1013,43 +1129,7 @@ impl GcState {
             counts[0] = counts[0].saturating_sub(1);
             self.sync_gen0_gauge(counts[0], None);
         }
-        // O(1) removal via the handle's cached `slot`. The cached position is
-        // only valid under the owning vector's lock; fall back to a pointer
-        // search if it's stale.
-        if handle.color.load(Ordering::Acquire) == color::Frozen {
-            let mut frozen = self.frozen.borrow_mut();
-            let slot = handle.slot.load(Ordering::Acquire);
-            if frozen
-                .get(slot)
-                .is_some_and(|h| HandleRc::ptr_eq(h, &handle))
-            {
-                swap_remove_handle(&mut frozen, slot);
-            } else {
-                remove_handle_by_ptr(&mut frozen, &handle);
-            }
-        } else {
-            let mut gens = self.generations.borrow_mut();
-            let g = usize::from(
-                handle
-                    .generation
-                    .load(Ordering::Acquire)
-                    .min(MAX_GENERATION),
-            );
-            let slot = handle.slot.load(Ordering::Acquire);
-            if gens[g]
-                .handles
-                .get(slot)
-                .is_some_and(|h| HandleRc::ptr_eq(h, &handle))
-            {
-                swap_remove_handle(&mut gens[g].handles, slot);
-            } else if !remove_handle_by_ptr(&mut gens[g].handles, &handle) {
-                for gg in 0..N_GENERATIONS {
-                    if gg != g && remove_handle_by_ptr(&mut gens[gg].handles, &handle) {
-                        break;
-                    }
-                }
-            }
-        }
+        drop(weak);
         serial_add(&self.tracked_count, usize::MAX);
         serial_add(&self.tracked_version, 1);
     }
@@ -1084,29 +1164,29 @@ impl GcState {
         }
     }
 
-    /// Drop the entries of objects that have died from `handles`, and from
-    /// the index; `budget` caps how many entries are examined, starting at
-    /// `start` (wrapping). Returns how many were dropped and where the
+    /// Drop the entries of objects that have died from generation `gen`;
+    /// `budget` caps how many entries are examined, starting at `start`
+    /// (wrapping). Returns how many were dropped and where the
     /// examination stopped.
-    // (An `IndexEntry` hashes and compares by its handle's immutable id.)
-    #[allow(clippy::mutable_key_type)]
     fn prune_dead_in(
-        index: &mut GcIndex,
-        handles: &mut Vec<HandleRc<TrackedHandle>>,
+        reg: &mut Registry,
+        gen: usize,
         start: usize,
         budget: usize,
     ) -> (usize, usize) {
         let mut removed = 0;
-        let mut i = if start < handles.len() { start } else { 0 };
+        let mut i = if start < reg.gens[gen].len() {
+            start
+        } else {
+            0
+        };
         let mut examined = 0;
-        while examined < budget && i < handles.len() {
+        while examined < budget && i < reg.gens[gen].len() {
             examined += 1;
-            if handles[i].object.is_dead() {
-                let h = handles.swap_remove(i);
-                index.remove(&h.id);
-                if let Some(moved) = handles.get(i) {
-                    moved.slot.store(i, Ordering::Release);
-                }
+            let slot = reg.gens[gen][i];
+            if reg.entry(slot).obj.as_ref().is_none_or(WeakObject::is_dead) {
+                // (The swap brings the list's last member to `i`.)
+                drop(reg.remove(slot));
                 removed += 1;
             } else {
                 i += 1;
@@ -1120,36 +1200,33 @@ impl GcState {
     /// Returns the young generation's live population.
     fn prune(&self, old_budget: usize) -> usize {
         let (removed, live) = {
-            let mut index = self.index.borrow_mut();
-            let mut gens = self.generations.borrow_mut();
-            let (mut removed, _) =
-                Self::prune_dead_in(&mut index, &mut gens[0].handles, 0, usize::MAX);
+            let mut reg = self.reg.borrow_mut();
+            let (mut removed, _) = Self::prune_dead_in(&mut reg, 0, 0, usize::MAX);
             // The older generations, a slice at a time: the cursor runs
             // over gen 1 then gen 2 as one sequence.
             let cursor = self.old_prune_cursor.load(Ordering::Relaxed);
-            let n1 = gens[1].handles.len();
+            let n1 = reg.gens[1].len();
             let (g, start) = if cursor < n1 {
                 (1, cursor)
             } else {
                 (2, cursor - n1)
             };
-            let (r, stop) =
-                Self::prune_dead_in(&mut index, &mut gens[g].handles, start, old_budget);
+            let (r, stop) = Self::prune_dead_in(&mut reg, g, start, old_budget);
             removed += r;
             self.old_deaths.fetch_add(r, Ordering::Relaxed);
-            let next = if stop >= gens[g].handles.len() {
+            let next = if stop >= reg.gens[g].len() {
                 if g == 1 {
-                    gens[1].handles.len()
+                    reg.gens[1].len()
                 } else {
                     0
                 }
             } else if g == 1 {
                 stop
             } else {
-                gens[1].handles.len() + stop
+                reg.gens[1].len() + stop
             };
             self.old_prune_cursor.store(next, Ordering::Relaxed);
-            (removed, gens[0].handles.len())
+            (removed, reg.gens[0].len())
         };
         if removed > 0 {
             self.tracked_count.fetch_sub(
@@ -1161,52 +1238,54 @@ impl GcState {
         live
     }
 
+    /// Whether the object of id `id` is registered. Only for a kind the id
+    /// index holds (not an instance: see [`Self::is_tracked_obj`]).
     pub fn is_tracked(&self, id: ObjectId) -> bool {
-        // Usually-miss probe: two relaxed loads instead of the index
+        // Usually-miss probe: two relaxed loads instead of the registry
         // borrow. A stale filter bit just takes the precise path.
         if !self.tracked_filter.may_contain(id) {
             return false;
         }
-        self.index.borrow().contains(&id)
+        self.reg.borrow().index.contains_key(&id)
     }
 
-    /// [`Self::is_tracked`] for an object that may be young (an instance,
-    /// function, generator, list or dict).
-    pub fn is_tracked_instance(&self, id: ObjectId) -> bool {
-        self.is_tracked(id) || self.is_young(id)
+    /// Whether `obj` is registered.
+    pub fn is_tracked_obj(&self, obj: &Object) -> bool {
+        self.reg
+            .borrow()
+            .slot_of(obj, &self.tracked_filter)
+            .is_some()
     }
 
-    /// O(1) handle lookup by object id (any generation or frozen).
-    pub fn handle_for(&self, id: ObjectId) -> Option<HandleRc<TrackedHandle>> {
-        if !self.tracked_filter.may_contain(id) {
-            return None;
-        }
-        self.index.borrow().get(&id).map(|e| e.0.clone())
+    /// [`Self::is_tracked_obj`] for an object that may be young (an
+    /// instance, function, generator, list or dict).
+    pub fn is_tracked_instance(&self, obj: &Object) -> bool {
+        self.is_tracked_obj(obj) || self.is_young(id_of(obj))
     }
 
     /// Snapshot every live tracked object that still carries an unrun
     /// `__del__`. The interpreter's shutdown pass walks this list to
-    /// finalize objects that are still alive at exit. The per-handle
-    /// `finalized` flag guarantees each `__del__` runs at most once.
-    pub fn finalization_candidates(&self) -> Vec<(HandleRc<TrackedHandle>, Object)> {
+    /// finalize objects that are still alive at exit; it claims each one
+    /// through [`Self::claim_finalizer`], so each `__del__` runs at most
+    /// once.
+    pub fn finalization_candidates(&self) -> Vec<Object> {
         self.flush_young();
+        let reg = self.reg.borrow();
         let mut out = Vec::new();
-        let mut consider = |h: &HandleRc<TrackedHandle>| {
+        let lists = reg.gens.iter().chain(std::iter::once(&reg.frozen));
+        for &slot in lists.flatten() {
+            let e = reg.entry(slot);
             // A finalizer already queued by a collection (but not yet
             // drained) must not be listed again.
-            if h.finalized.load(Ordering::Acquire) || h.finalize_queued.load(Ordering::Acquire) {
-                return;
+            if e.has(F_FINALIZED) || e.has(F_QUEUED) {
+                continue;
             }
-            if let Some(obj) = h.object.upgrade() {
+            if let Some(obj) = e.obj.as_ref().and_then(WeakObject::upgrade) {
                 if has_finalizer(&obj) {
-                    out.push((h.clone(), obj));
+                    out.push(obj);
                 }
             }
-        };
-        for gen in self.generations.borrow().iter() {
-            gen.handles.iter().for_each(&mut consider);
         }
-        self.frozen.borrow().iter().for_each(&mut consider);
         out
     }
 
@@ -1383,12 +1462,7 @@ impl GcState {
     /// Total population (across all generations + frozen).
     pub fn population(&self) -> usize {
         self.flush_young();
-        let gens = self.generations.borrow();
-        let mut n = 0;
-        for g in gens.iter() {
-            n += g.handles.len();
-        }
-        n + self.frozen.borrow().len()
+        self.reg.borrow().len()
     }
 
     /// Snapshot all live tracked objects. Used by
@@ -1397,17 +1471,17 @@ impl GcState {
         // `gc.get_objects()` enumerates every container CPython tracks,
         // including the all-scalar ones whose tracking we defer.
         self.promote_all_deferred();
-        let gens = self.generations.borrow();
+        let reg = self.reg.borrow();
         let mut out = Vec::new();
-        let mut push = |h: &HandleRc<TrackedHandle>| out.extend(h.object.upgrade());
+        let mut push = |s: &u32| {
+            out.extend(reg.entry(*s).obj.as_ref().and_then(WeakObject::upgrade));
+        };
         match generation {
-            Some(g) if g < N_GENERATIONS => gens[g].handles.iter().for_each(&mut push),
-            _ => gens
-                .iter()
-                .for_each(|g| g.handles.iter().for_each(&mut push)),
+            Some(g) if g < N_GENERATIONS => reg.gens[g].iter().for_each(&mut push),
+            _ => reg.gens.iter().for_each(|g| g.iter().for_each(&mut push)),
         }
         if generation.is_none() {
-            self.frozen.borrow().iter().for_each(&mut push);
+            reg.frozen.iter().for_each(&mut push);
         }
         out
     }
@@ -1416,13 +1490,14 @@ impl GcState {
     /// frozen so it is ignored by future collections.
     pub fn freeze_all(&self) {
         self.flush_young();
-        let mut gens = self.generations.borrow_mut();
-        let mut frozen = self.frozen.borrow_mut();
-        for g in gens.iter_mut() {
-            for h in g.handles.drain(..) {
-                h.color.store(color::Frozen, Ordering::Release);
-                h.slot.store(frozen.len(), Ordering::Release);
-                frozen.push(h);
+        let mut reg = self.reg.borrow_mut();
+        for g in 0..N_GENERATIONS {
+            for slot in std::mem::take(&mut reg.gens[g]) {
+                let pos = reg.frozen.len() as u32;
+                let e = &mut reg.slab[slot as usize];
+                e.gen = GEN_FROZEN;
+                e.pos = pos;
+                reg.frozen.push(slot);
             }
         }
         self.tracked_version.fetch_add(1, Ordering::AcqRel);
@@ -1431,20 +1506,19 @@ impl GcState {
     /// `gc.unfreeze()` — move every frozen object back to
     /// generation 0.
     pub fn unfreeze_all(&self) {
-        // Lock order: generations before frozen, matching `freeze_all`.
-        let mut gens = self.generations.borrow_mut();
-        let mut frozen = self.frozen.borrow_mut();
-        for h in frozen.drain(..) {
-            h.color.store(color::White, Ordering::Release);
-            h.generation.store(0, Ordering::Release);
-            h.slot.store(gens[0].handles.len(), Ordering::Release);
-            gens[0].handles.push(h);
+        let mut reg = self.reg.borrow_mut();
+        for slot in std::mem::take(&mut reg.frozen) {
+            let pos = reg.gens[0].len() as u32;
+            let e = &mut reg.slab[slot as usize];
+            e.gen = 0;
+            e.pos = pos;
+            reg.gens[0].push(slot);
         }
         self.tracked_version.fetch_add(1, Ordering::AcqRel);
     }
 
     pub fn freeze_count(&self) -> usize {
-        self.frozen.borrow().len()
+        self.reg.borrow().frozen.len()
     }
 
     /// Collect generations `0..=upto`. Returns the number of
@@ -1544,10 +1618,16 @@ impl GcState {
             self.sync_gen0_gauge(counts[0], None);
         }
         // The tracked-id filter only ever gains bits; once most of them
-        // name objects long gone, rebuild it from the live index.
-        let live = self.index.borrow().len();
+        // name objects long gone, rebuild it from the live index (sooner
+        // after a full collection, which finds the most dead).
+        let live = self.reg.borrow().index.len();
         let stale = self.tracked_filter.inserts_since_rebuild();
-        if gen == N_GENERATIONS - 1 || stale > 4096.max(live.saturating_mul(4)) {
+        let bound = if gen == N_GENERATIONS - 1 {
+            live
+        } else {
+            live.saturating_mul(4)
+        };
+        if stale > 4096.max(bound) {
             self.rebuild_tracked_filter();
         }
         self.collecting.store(false, Ordering::Release);
@@ -1560,8 +1640,8 @@ impl GcState {
     /// Rebuild the tracked-id miss filter from the live index. Holding the
     /// index borrow serializes it against every `track`.
     pub fn rebuild_tracked_filter(&self) {
-        let index = self.index.borrow();
-        self.tracked_filter.rebuild(index.iter().map(|e| e.0.id));
+        let reg = self.reg.borrow();
+        self.tracked_filter.rebuild(reg.index.keys().copied());
     }
 
     /// Collect a specific generation. Used by [`Self::collect`].
@@ -1574,15 +1654,24 @@ impl GcState {
     /// `weakref_cb` without the destructive teardown of a full collection
     /// (RFC 0040: `test_shutdown`).
     fn collect_generation(&self, gen: usize, weakref_only: bool) -> usize {
-        // Phase 1: the live objects of this generation and the younger ones,
-        // each held strongly for the collection's duration. Entries whose
-        // object died are dropped on the way.
-        let mut cands = self.snapshot_for_collection(gen);
-        let n_real = cands.len();
-        if n_real == 0 {
+        let gen = gen.min(N_GENERATIONS - 1);
+        // Phase 1: mark the live entries of this generation and the
+        // younger ones as candidates. Entries whose object died are dropped
+        // on the way. The candidates are the generation lists themselves,
+        // and each one's marking state lives in its entry (as CPython's
+        // lives in the object's header): the collection takes no reference
+        // and makes no copy per candidate.
+        if self.mark_candidates(gen) == 0 {
             return 0;
         }
-        let mut by_id: GcIdMap = cands.iter().enumerate().map(|(i, c)| (c.id, i)).collect();
+        // Temporary candidates, and their positions by id.
+        let mut temps: Vec<Temp> = Vec::new();
+
+        // The registry can't change while the mark phase reads it: no code
+        // runs, and a dying object's own removal finds it busy.
+        let reg = self.reg.borrow();
+        let finder = Finder::new(&reg, &self.tracked_filter);
+        let lists = &reg.gens[..=gen];
 
         // Phase 2: one walk over the candidates promotes the untracked nodes
         // reachable from them to temporary candidates for this pass only,
@@ -1594,97 +1683,184 @@ impl GcState {
         // internal edges are accounted. A temporary is a candidate from its
         // first edge on, so each of its edges is tallied too. Temporaries
         // take part in the mark walk but are never reclaimed or tracked.
-        let mut scanned = 0usize;
         let mut found: Vec<Object> = Vec::new();
-        while scanned < cands.len() {
-            let parent_is_iter = matches!(cands[scanned].obj, Object::Iter(_));
-            let parent_is_frame = matches!(cands[scanned].obj, Object::Frame(_));
-            traverse_object(&cands[scanned].obj, &mut |child| {
+        let tally = |obj: &Object, temps: &mut Vec<Temp>, found: &mut Vec<Object>| {
+            let parent_is_iter = matches!(obj, Object::Iter(_));
+            let parent_is_frame = matches!(obj, Object::Frame(_));
+            traverse_collected(obj, &mut |child| {
                 if child.never_gc_candidate() {
                     return;
                 }
-                if let Some(&i) = by_id.get(&id_of(child)) {
-                    let t = &cands[i];
-                    t.gc_refs.set(t.gc_refs.get() - 1);
-                } else if promotes_temporarily(child, parent_is_iter, parent_is_frame) {
-                    found.push(child.clone());
+                match finder.find(child) {
+                    Some(Hit::Real(s)) => {
+                        let e = reg.entry(s);
+                        e.gc_refs.set(e.gc_refs.get().wrapping_sub(1));
+                    }
+                    Some(Hit::Temp(i)) => {
+                        let t = &temps[i as usize];
+                        t.gc_refs.set(t.gc_refs.get() - 1);
+                    }
+                    None => {
+                        if promotes_temporarily(child, parent_is_iter, parent_is_frame) {
+                            found.push(child.clone());
+                        }
+                    }
                 }
             });
             // (One entry per edge: a node reached twice is promoted once and
             // tallied twice.)
             for child in found.drain(..) {
-                let cid = id_of(&child);
-                let i = match by_id.entry(cid) {
-                    std::collections::hash_map::Entry::Vacant(e) => {
-                        e.insert(cands.len());
-                        cands.push(Cand::new(child, cid, None));
-                        cands.len() - 1
+                let i = match finder.find(&child) {
+                    Some(Hit::Temp(i)) => i as usize,
+                    Some(Hit::Real(s)) => {
+                        let e = reg.entry(s);
+                        e.gc_refs.set(e.gc_refs.get().wrapping_sub(1));
+                        continue;
+                    }
+                    None => {
+                        finder.add_temp(&child, temps.len());
+                        temps.push(Temp {
+                            obj: child,
+                            gc_refs: std::cell::Cell::new(0),
+                            reached: std::cell::Cell::new(false),
+                        });
+                        temps.len() - 1
                     }
-                    std::collections::hash_map::Entry::Occupied(e) => *e.get(),
                 };
-                let t = &cands[i];
+                let t = &temps[i];
                 t.gc_refs.set(t.gc_refs.get() - 1);
             }
+        };
+        for &slot in lists.iter().flatten() {
+            // SAFETY: the mark walk runs no code that could release the
+            // object (see `traverse_collected`).
+            unsafe {
+                reg.entry(slot)
+                    .obj
+                    .as_ref()
+                    .and_then(|w| w.with_borrowed(|obj| tally(obj, &mut temps, &mut found)));
+            }
+        }
+        let mut scanned = 0;
+        while scanned < temps.len() {
+            let obj = temps[scanned].obj.clone();
+            tally(&obj, &mut temps, &mut found);
             scanned += 1;
         }
+        let lookup = |child: &Object| finder.find(child);
 
         // Phase 3: add the outer refcount, read after the walk (an iterator
         // synthesises a fresh wrapper for its buffer on each traverse, alive
-        // only during the visit). Each candidate holds one reference of its
+        // only during the visit). A temporary holds one reference of its
         // own.
-        for c in &cands {
-            c.gc_refs
-                .set(strong_count_for(&c.obj) as i64 - 1 + c.gc_refs.get());
-        }
-
-        // Phase 4: anything with gc_refs > 0 is reachable from outside; mark
-        // it black and propagate.
-        let mut grey: Vec<usize> = Vec::new();
-        for (i, c) in cands.iter().enumerate() {
-            if c.gc_refs.get() > 0 {
-                c.color.set(color::Grey);
-                grey.push(i);
+        let mut grey: Vec<u32> = Vec::new();
+        for &slot in lists.iter().flatten() {
+            let e = reg.entry(slot);
+            let strong = e.obj.as_ref().map_or(0, WeakObject::strong_count);
+            let refs = i64::from(e.gc_refs.get()) + strong as i64;
+            e.gc_refs
+                .set(refs.clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32);
+            // Phase 4: anything referenced from outside is reachable; mark
+            // it and propagate.
+            if refs > 0 {
+                e.set(F_REACHED, true);
+                grey.push(slot);
+            }
+        }
+        for (i, t) in temps.iter().enumerate() {
+            t.gc_refs
+                .set(strong_count_for(&t.obj) as i64 - 1 + t.gc_refs.get());
+            if t.gc_refs.get() > 0 {
+                t.reached.set(true);
+                grey.push(i as u32 | TEMP_BIT);
             }
         }
         if root_debug() {
-            for &i in &grey {
-                let c = &cands[i];
-                eprintln!(
-                    "[root] id={} gc_refs={} sc={} {}",
-                    c.id,
-                    c.gc_refs.get(),
-                    strong_count_for(&c.obj),
-                    c.obj.type_name_owned()
-                );
+            for &g in &grey {
+                if g & TEMP_BIT == 0 {
+                    let e = reg.entry(g);
+                    eprintln!(
+                        "[root] slot={} gc_refs={} sc={}",
+                        g,
+                        e.gc_refs.get(),
+                        e.obj.as_ref().map_or(0, WeakObject::strong_count)
+                    );
+                } else {
+                    let t = &temps[(g & !TEMP_BIT) as usize];
+                    eprintln!(
+                        "[root] temp gc_refs={} {}",
+                        t.gc_refs.get(),
+                        t.obj.type_name_owned()
+                    );
+                }
             }
         }
-        Self::propagate_black(&cands, &by_id, &mut grey);
+        if stats_debug() {
+            let mut kinds: std::collections::BTreeMap<String, usize> = Default::default();
+            for t in &temps {
+                *kinds.entry(t.obj.type_name_owned()).or_default() += 1;
+            }
+            eprintln!(
+                "[gc] gen={} cands={} roots={} temps={} {:?}",
+                gen,
+                lists.iter().map(Vec::len).sum::<usize>(),
+                grey.len(),
+                temps.len(),
+                kinds
+            );
+        }
+        Self::propagate_reached(&reg, &temps, &lookup, &mut grey);
 
-        // Phase 5: white real candidates are unreachable cyclic garbage.
-        let unreachable: Vec<usize> = (0..n_real).filter(|&i| cands[i].is_white()).collect();
+        // Phase 5: the candidates still unreached are unreachable cyclic
+        // garbage; hold each for the rest of the collection.
+        let mut unreachable: Vec<(u32, Object)> = Vec::new();
+        for &slot in lists.iter().flatten() {
+            let e = reg.entry(slot);
+            if !e.has(F_REACHED) {
+                if let Some(obj) = e.obj.as_ref().and_then(WeakObject::upgrade) {
+                    unreachable.push((slot, obj));
+                }
+            }
+        }
 
         // CPython's `handle_weakrefs`: a weakref that is *itself* part of the
         // cyclic trash has its callback cleared without invocation — only
         // weakrefs rooted outside the dying subgraph observe the deaths
         // (test_callbacks_on_callback).
-        let trash_ids: IdSet = unreachable.iter().map(|&i| cands[i].id).collect();
+        // (Built only for a collection whose garbage was weakly referenced.)
+        let trash_ids: std::cell::OnceCell<IdSet> = std::cell::OnceCell::new();
+        let is_trash = |id: ObjectId| {
+            trash_ids
+                .get_or_init(|| unreachable.iter().map(|(_, o)| id_of(o)).collect())
+                .contains(&id)
+        };
         let wrapper_is_trash = |slot: &crate::sync::Rc<crate::weakref_registry::WeakRefSlot>| {
             slot.py_ref
                 .borrow()
                 .as_ref()
                 .and_then(crate::sync::Weak::upgrade)
-                .is_none_or(|inst| {
-                    trash_ids.contains(&(crate::sync::Rc::as_ptr(&inst) as usize as u64))
-                })
+                .is_none_or(|inst| is_trash(crate::sync::Rc::as_ptr(&inst) as usize as u64))
+        };
+        // The weakrefs to `obj`, cleared (a lock-free probe answers for an
+        // object never weakly referenced).
+        let notify_clear = |obj: &Object| {
+            let id = id_of(obj);
+            if crate::weakref_registry::may_have_weakrefs(id) {
+                crate::weakref_registry::notify_clear(id)
+            } else {
+                Vec::new()
+            }
         };
+        let pending_finalizer =
+            |slot: u32, obj: &Object| has_finalizer(obj) && !reg.entry(slot).has(F_FINALIZED);
 
         if weakref_only {
             let mut weakref_callbacks = Vec::new();
-            for &i in &unreachable {
-                if cands[i].pending_finalizer() {
+            for (slot, obj) in &unreachable {
+                if pending_finalizer(*slot, obj) {
                     continue;
                 }
-                for (slot, cb) in crate::weakref_registry::notify_clear(cands[i].id) {
+                for (slot, cb) in notify_clear(obj) {
                     if let Some(cb) = cb {
                         if !wrapper_is_trash(&slot) {
                             weakref_callbacks.push((slot, cb));
@@ -1692,7 +1868,13 @@ impl GcState {
                     }
                 }
             }
+            for &slot in lists.iter().flatten() {
+                let e = reg.entry(slot);
+                e.set(F_CAND | F_REACHED, false);
+            }
+            drop(reg);
             queue_weakref_callbacks(weakref_callbacks, |_| false);
+            drop(unreachable);
             return 0;
         }
 
@@ -1701,8 +1883,8 @@ impl GcState {
         // watching an object a finalizer later revives stays cleared
         // (`test_io.test_garbage_collection`).
         let mut weakref_callbacks = Vec::new();
-        for &i in &unreachable {
-            for (slot, cb) in crate::weakref_registry::notify_clear(cands[i].id) {
+        for (_, obj) in &unreachable {
+            for (slot, cb) in notify_clear(obj) {
                 if let Some(cb) = cb {
                     if !wrapper_is_trash(&slot) {
                         weakref_callbacks.push((slot, cb));
@@ -1716,167 +1898,320 @@ impl GcState {
         // the collection) may resurrect them, and CPython only counts
         // objects it actually reclaims. The next collection, by which point
         // the finalizer has set `finalized`, reclaims the ones it didn't.
-        let (deferred, maybe_dead): (Vec<usize>, Vec<usize>) = unreachable
-            .iter()
-            .partition(|&&i| cands[i].pending_finalizer());
-        for &i in &deferred {
-            if let Some(e) = &cands[i].entry {
-                if !e.finalize_queued.swap(true, Ordering::AcqRel) {
-                    run_finalizer(&cands[i].obj);
-                }
-            }
-        }
         // CPython runs `finalize_garbage` before `delete_garbage`, so a
         // pending finalizer always sees its own class, closure cells, and
         // referents intact: protect the deferred objects' whole subgraphs.
-        let mut protect: Vec<usize> = Vec::new();
-        for &i in &deferred {
-            cands[i].color.set(color::Grey);
-            protect.push(i);
+        let mut protect: Vec<u32> = Vec::new();
+        for (slot, obj) in &unreachable {
+            if pending_finalizer(*slot, obj) {
+                let e = reg.entry(*slot);
+                if !e.has(F_QUEUED) {
+                    e.set(F_QUEUED, true);
+                    run_finalizer(obj);
+                }
+                e.set(F_REACHED, true);
+                protect.push(*slot);
+            }
         }
-        Self::propagate_black(&cands, &by_id, &mut protect);
+        Self::propagate_reached(&reg, &temps, &lookup, &mut protect);
 
-        let dead: Vec<usize> = maybe_dead
-            .into_iter()
-            .filter(|&i| cands[i].is_white())
+        let dead: Vec<Object> = unreachable
+            .iter()
+            .filter(|(slot, _)| !reg.entry(*slot).has(F_REACHED))
+            .map(|(_, obj)| obj.clone())
             .collect();
         let collected = dead.len();
         // Temporarily promoted iterators and immutable containers that ended
-        // up white are cyclic garbage too, freed by refcount once the cycle's
-        // mutable anchor is cleared below. CPython counts each (`test_tuple`
-        // asserts the closing tuple is counted alongside its list).
+        // up unreached are cyclic garbage too, freed by refcount once the
+        // cycle's mutable anchor is cleared below. CPython counts each
+        // (`test_tuple` asserts the closing tuple is counted alongside its
+        // list).
         let reported = collected
-            + cands[n_real..]
+            + temps
                 .iter()
-                .filter(|c| {
-                    c.is_white()
+                .filter(|t| {
+                    !t.reached.get()
                         && matches!(
-                            c.obj,
+                            t.obj,
                             Object::Iter(_) | Object::Tuple(_) | Object::FrozenSet(_)
                         )
                 })
                 .count();
+        // The clears below release objects, whose removal from the registry
+        // must find it free.
+        drop(reg);
 
         // Break the cycles by clearing the reclaimed objects' fields — or,
         // under `gc.DEBUG_SAVEALL`, park them in `gc.garbage` intact.
         if self.debug.load(Ordering::Acquire) & DEBUG_SAVEALL != 0 {
             let mut garbage = self.garbage.borrow_mut();
-            for &i in &dead {
-                garbage.push(cands[i].obj.clone());
+            for obj in &dead {
+                garbage.push(obj.clone());
             }
         } else {
             // Instances whose `__dict__` is shared are deferred; once every
             // other dead object has released its references, a dict held
             // only by dead holders is down to one owner and the retry
             // clears it (a live holder keeps it intact).
-            let mut shared_dict_holders: Vec<usize> = Vec::new();
-            for &i in &dead {
-                if !clear_object_fields(&cands[i].obj) {
-                    shared_dict_holders.push(i);
+            let mut shared_dict_holders: Vec<&Object> = Vec::new();
+            for obj in &dead {
+                if !clear_object_fields(obj) {
+                    shared_dict_holders.push(obj);
                 }
             }
-            for i in shared_dict_holders {
-                clear_object_fields(&cands[i].obj);
+            for obj in shared_dict_holders {
+                clear_object_fields(obj);
             }
         }
 
         // Queue the weakref callbacks (after finalisers and cyclic clears,
         // matching CPython's order) for the interpreter's next safe point.
-        queue_weakref_callbacks(weakref_callbacks, |slot| {
-            slot.py_ref
-                .borrow()
-                .as_ref()
-                .and_then(crate::sync::Weak::upgrade)
-                .is_none_or(|inst| {
-                    trash_ids.contains(&(crate::sync::Rc::as_ptr(&inst) as usize as u64))
-                })
-        });
+        queue_weakref_callbacks(weakref_callbacks, wrapper_is_trash);
 
         // Phase 6: rebuild the generation lists. Survivors of generation
         // `g` move to generation min(g+1, N_GENERATIONS-1); the dead leave
-        // the index. Dropping `cands` then frees the dead by refcount.
-        self.rebuild_generations(gen, &cands[..n_real]);
+        // the registry. Dropping the held objects then frees the dead by
+        // refcount.
+        self.rebuild_generations(gen);
         self.tracked_count.fetch_sub(
             collected.min(self.tracked_count.load(Ordering::Acquire)),
             Ordering::AcqRel,
         );
         self.tracked_version.fetch_add(1, Ordering::AcqRel);
-        drop(cands);
+        drop(dead);
+        drop(unreachable);
+        drop(temps);
         reported
     }
 
-    /// Blacken everything reachable from the grey candidates in `grey`.
-    fn propagate_black(cands: &[Cand], by_id: &GcIdMap, grey: &mut Vec<usize>) {
-        while let Some(i) = grey.pop() {
-            cands[i].color.set(color::Black);
-            traverse_object(&cands[i].obj, &mut |child| {
+    /// Mark everything reachable from the candidates on the `grey` stack
+    /// (registry slots, or temporaries tagged [`TEMP_BIT`]), which are
+    /// marked already; `lookup` finds a child's candidate.
+    fn propagate_reached(
+        reg: &Registry,
+        temps: &[Temp],
+        lookup: &dyn Fn(&Object) -> Option<Hit>,
+        grey: &mut Vec<u32>,
+    ) {
+        while let Some(item) = grey.pop() {
+            let mut visit = |child: &Object| {
                 if child.never_gc_candidate() {
                     return;
                 }
-                if let Some(&j) = by_id.get(&id_of(child)) {
-                    if cands[j].is_white() {
-                        cands[j].color.set(color::Grey);
-                        grey.push(j);
+                match lookup(child) {
+                    Some(Hit::Real(s)) => {
+                        let e = reg.entry(s);
+                        if !e.has(F_REACHED) {
+                            e.set(F_REACHED, true);
+                            grey.push(s);
+                        }
                     }
+                    Some(Hit::Temp(i)) => {
+                        let t = &temps[i as usize];
+                        if !t.reached.get() {
+                            t.reached.set(true);
+                            grey.push(i | TEMP_BIT);
+                        }
+                    }
+                    None => {}
                 }
-            });
+            };
+            if item & TEMP_BIT == 0 {
+                // SAFETY: as in the mark walk.
+                unsafe {
+                    reg.entry(item)
+                        .obj
+                        .as_ref()
+                        .and_then(|w| w.with_borrowed(|obj| traverse_collected(obj, &mut visit)));
+                }
+            } else {
+                traverse_collected(&temps[(item & !TEMP_BIT) as usize].obj, &mut visit);
+            }
         }
     }
 
-    /// The live objects of generations `0..=upto` as collection candidates;
-    /// entries whose object died are dropped from those generations and the
-    /// index.
-    fn snapshot_for_collection(&self, upto: usize) -> Vec<Cand> {
-        let mut index = self.index.borrow_mut();
-        let mut gens = self.generations.borrow_mut();
-        let mut out = Vec::new();
-        let mut pruned = 0usize;
-        for g in gens.iter_mut().take(upto.min(N_GENERATIONS - 1) + 1) {
-            g.handles.retain(|h| match h.object.upgrade() {
-                Some(obj) => {
-                    out.push(Cand::new(obj, h.id, Some(h.clone())));
-                    true
-                }
-                None => {
-                    index.remove(&h.id);
-                    pruned += 1;
-                    false
+    /// Mark the live entries of generations `0..=upto` as candidates of a
+    /// collection, dropping the entries whose object died. Returns how
+    /// many there are.
+    fn mark_candidates(&self, upto: usize) -> usize {
+        let mut reg = self.reg.borrow_mut();
+        let mut dead = Vec::new();
+        let mut n = 0;
+        for g in 0..=upto {
+            let mut i = 0;
+            while i < reg.gens[g].len() {
+                let slot = reg.gens[g][i];
+                let e = reg.entry(slot);
+                if e.obj.as_ref().is_none_or(WeakObject::is_dead) {
+                    // (The swap brings the list's last member to `i`.)
+                    dead.push(reg.remove(slot));
+                    continue;
                 }
-            });
-            for (i, h) in g.handles.iter().enumerate() {
-                h.slot.store(i, Ordering::Release);
+                e.gc_refs.set(0);
+                e.set(F_REACHED, false);
+                e.set(F_CAND, true);
+                i += 1;
             }
+            n += reg.gens[g].len();
         }
-        if pruned > 0 {
+        drop(reg);
+        if !dead.is_empty() {
             self.tracked_count.fetch_sub(
-                pruned.min(self.tracked_count.load(Ordering::Acquire)),
+                dead.len().min(self.tracked_count.load(Ordering::Acquire)),
                 Ordering::AcqRel,
             );
         }
-        out
+        n
+    }
+
+    /// Move the surviving candidates of generations `0..=upto` up a
+    /// generation and drop the dead (unreached) ones from the registry. A
+    /// member that joined during the collection stays where it is.
+    fn rebuild_generations(&self, upto: usize) {
+        let mut reg = self.reg.borrow_mut();
+        let mut lists: [Vec<u32>; N_GENERATIONS] = Default::default();
+        let mut dead = Vec::new();
+        for g in 0..=upto {
+            let target = (g + 1).min(N_GENERATIONS - 1);
+            for slot in std::mem::take(&mut reg.gens[g]) {
+                let e = reg.entry(slot);
+                if !e.has(F_CAND) {
+                    lists[g].push(slot);
+                    continue;
+                }
+                let reached = e.has(F_REACHED);
+                e.set(F_CAND | F_REACHED, false);
+                if reached {
+                    reg.slab[slot as usize].gen = target as u8;
+                    lists[target].push(slot);
+                } else {
+                    // (Already out of its list.)
+                    reg.forget_key(slot);
+                    dead.push(reg.release(slot));
+                }
+            }
+        }
+        for (g, list) in lists.into_iter().enumerate() {
+            if g > upto && list.is_empty() {
+                continue;
+            }
+            let start = reg.gens[g].len();
+            for (i, &slot) in list.iter().enumerate() {
+                reg.slab[slot as usize].pos = (start + i) as u32;
+            }
+            reg.gens[g].extend(list);
+        }
+        drop(reg);
+        drop(dead);
+    }
+}
+
+/// How a collection's mark walk finds a child's candidate: the registry
+/// (which can't change while the walk runs), a cache in front of its id
+/// index for the children seen again and again (a module's namespace, a
+/// shared function), and the temporaries.
+struct Finder<'a> {
+    reg: &'a Registry,
+    filter: &'a crate::hot_filter::RebuildableBloom,
+    /// Temporaries' positions by id.
+    temp_ids: std::cell::RefCell<GcIdMap>,
+    /// Some temporary is also a registered object (outside the
+    /// candidates): a registered child may then still be a temporary.
+    registered_temp: std::cell::Cell<bool>,
+    /// Payload address -> slot (or [`NO_SLOT`]), direct-mapped.
+    cache_keys: [std::cell::Cell<ObjectId>; FINDER_CACHE],
+    cache_slots: [std::cell::Cell<u32>; FINDER_CACHE],
+}
+
+/// [`Finder`]'s cache size.
+const FINDER_CACHE: usize = 256;
+
+impl<'a> Finder<'a> {
+    fn new(reg: &'a Registry, filter: &'a crate::hot_filter::RebuildableBloom) -> Self {
+        Self {
+            reg,
+            filter,
+            temp_ids: std::cell::RefCell::new(GcIdMap::default()),
+            registered_temp: std::cell::Cell::new(false),
+            cache_keys: std::array::from_fn(|_| std::cell::Cell::new(0)),
+            cache_slots: std::array::from_fn(|_| std::cell::Cell::new(NO_SLOT)),
+        }
     }
 
-    fn rebuild_generations(&self, upto: usize, cands: &[Cand]) {
-        // Lock order MUST match `track` (index before generations).
-        let mut index = self.index.borrow_mut();
-        let mut gens = self.generations.borrow_mut();
-        for g in 0..=upto.min(N_GENERATIONS - 1) {
-            gens[g].handles.clear();
+    /// `obj`'s registry slot, if it has one.
+    #[inline]
+    fn slot(&self, obj: &Object) -> Option<u32> {
+        if let Some(cell) = intrusive_slot(obj) {
+            let s = cell.get();
+            return (s != NO_SLOT).then_some(s);
         }
-        for c in cands {
-            let Some(h) = &c.entry else { continue };
-            if c.is_white() {
-                index.remove(&h.id);
-                continue;
+        let key = obj.payload_addr()? as ObjectId;
+        if !self.filter.may_contain(key) {
+            return None;
+        }
+        let i = ((key >> 4) ^ (key >> 12)) as usize & (FINDER_CACHE - 1);
+        let slot = if self.cache_keys[i].get() == key {
+            self.cache_slots[i].get()
+        } else {
+            let slot = self.reg.index.get(&key).copied().unwrap_or(NO_SLOT);
+            self.cache_keys[i].set(key);
+            self.cache_slots[i].set(slot);
+            slot
+        };
+        (slot != NO_SLOT).then_some(slot)
+    }
+
+    /// `child`'s candidate, if it is one.
+    #[inline]
+    fn find(&self, child: &Object) -> Option<Hit> {
+        if let Some(slot) = self.slot(child) {
+            if self.reg.entry(slot).has(F_CAND) {
+                return Some(Hit::Real(slot));
             }
-            let g = h.generation.load(Ordering::Acquire);
-            let new_g = g.saturating_add(1).min(MAX_GENERATION);
-            h.generation.store(new_g, Ordering::Release);
-            let new_g = usize::from(new_g);
-            h.slot.store(gens[new_g].handles.len(), Ordering::Release);
-            gens[new_g].handles.push(h.clone());
+            if !self.registered_temp.get() {
+                return None;
+            }
+        }
+        if !may_be_temporary(child) {
+            return None;
+        }
+        let ids = self.temp_ids.borrow();
+        if ids.is_empty() {
+            return None;
         }
+        ids.get(&id_of(child)).map(|&i| Hit::Temp(i as u32))
     }
+
+    /// Record `child` as the temporary at position `i`.
+    fn add_temp(&self, child: &Object, i: usize) {
+        if self.slot(child).is_some() {
+            self.registered_temp.set(true);
+        }
+        self.temp_ids.borrow_mut().insert(id_of(child), i);
+    }
+}
+
+/// Whether [`promotes_temporarily`] can accept `child` (from some parent).
+#[inline]
+fn may_be_temporary(child: &Object) -> bool {
+    matches!(
+        child,
+        Object::Iter(_)
+            | Object::Tuple(_)
+            | Object::FrozenSet(_)
+            | Object::DictView(_)
+            | Object::Slice(_)
+            | Object::Cell(_)
+            | Object::Traceback(_)
+            | Object::Frame(_)
+            | Object::BoundMethod(_)
+            | Object::StaticMethod(_)
+            | Object::ClassMethod(_)
+            | Object::Property(_)
+            | Object::List(_)
+            | Object::Instance(_)
+            | Object::Dict(_)
+    )
 }
 
 /// Id → candidate position, for one collection.
@@ -1893,7 +2228,6 @@ fn promotes_temporarily(child: &Object, parent_is_iter: bool, parent_is_frame: b
         // l.append(t)`, a recursive closure's `function -> cell ->
         // function`, an exception's `__traceback__ -> frame -> f_locals`).
         Object::Iter(_)
-        | Object::Tuple(_)
         | Object::FrozenSet(_)
         | Object::DictView(_)
         | Object::Slice(_)
@@ -1904,6 +2238,10 @@ fn promotes_temporarily(child: &Object, parent_is_iter: bool, parent_is_frame: b
         | Object::StaticMethod(_)
         | Object::ClassMethod(_)
         | Object::Property(_) => true,
+        // A tuple of atomic values can't take part in a cycle (CPython
+        // untracks it at creation), and a list of edges or a dict of
+        // coordinates holds thousands.
+        Object::Tuple(t) => t.iter().any(|x| !is_atomic(x)),
         // Lists are tracked at creation, so an *untracked* list is only an
         // iterator's private snapshot buffer.
         Object::List(_) => parent_is_iter,
@@ -2043,12 +2381,41 @@ pub fn strong_count_for(obj: &Object) -> usize {
 /// refcount-seeded reachability model — an unvisited child keeps its
 /// external `gc_refs` and therefore survives the pass.
 pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) {
+    traverse_impl::<false>(obj, visit);
+}
+
+/// [`traverse_object`] for a collection's mark walk, whose visitor runs no
+/// code and borrows no container: the common containers' cells are read
+/// without borrow bookkeeping while nothing holds them mutably, and an
+/// instance's class is visited through a handle that takes no reference.
+fn traverse_collected(obj: &Object, visit: &mut dyn FnMut(&Object)) {
+    traverse_impl::<true>(obj, visit);
+}
+
+/// Run `f` on `cell`'s value: a guard-free view when `PEEK` allows one,
+/// else a shared borrow (`None` while the cell is borrowed mutably).
+#[inline(always)]
+fn with_read<const PEEK: bool, T, R>(cell: &RefCell<T>, f: impl FnOnce(&T) -> R) -> Option<R> {
+    if PEEK {
+        // SAFETY: a `PEEK` traversal's visitor runs no code and borrows no
+        // container (see `traverse_collected`), so nothing can borrow the
+        // cell mutably while the view lives.
+        if let Some(v) = unsafe { cell.peek() } {
+            return Some(f(v));
+        }
+    }
+    cell.try_borrow().ok().map(|g| f(&g))
+}
+
+#[inline(always)]
+fn traverse_impl<const PEEK: bool>(obj: &Object, visit: &mut dyn FnMut(&Object)) {
     match obj {
         Object::List(l) => {
-            let Ok(v) = l.try_borrow() else { return };
-            for item in v.iter() {
-                visit(item);
-            }
+            with_read::<PEEK, _, _>(l, |v| {
+                for item in v.iter() {
+                    visit(item);
+                }
+            });
         }
         Object::Tuple(t) => {
             for item in t.iter() {
@@ -2056,18 +2423,20 @@ pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) {
             }
         }
         Object::Dict(d) | Object::MappingProxy(d) | Object::SimpleNamespace(d) => {
-            let Ok(m) = d.try_borrow() else { return };
-            for (k, v) in m.iter() {
-                visit(&k.0);
-                visit(v);
-            }
+            with_read::<PEEK, _, _>(d, |m| {
+                for (k, v) in m.iter() {
+                    visit(&k.0);
+                    visit(v);
+                }
+            });
         }
         Object::MappingProxyObj(inner) => visit(inner),
         Object::Set(s) => {
-            let Ok(m) = s.try_borrow() else { return };
-            for k in m.iter() {
-                visit(&k.0);
-            }
+            with_read::<PEEK, _, _>(s, |m| {
+                for k in m.iter() {
+                    visit(&k.0);
+                }
+            });
         }
         Object::FrozenSet(s) => {
             for k in s.iter() {
@@ -2081,9 +2450,23 @@ pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) {
             // instances (e.g. `A.a = A(); del A`) must see that edge subtracted
             // or it never collects. Built-in types are immortal and untracked,
             // so skip them (the `by_id` lookup would miss anyway).
-            let cls = i.cls();
-            if !cls.flags.is_builtin {
-                visit(&Object::Type(cls));
+            // SAFETY: as `with_read`'s; the instance keeps its class alive,
+            // and the view, which owns no reference, is never dropped.
+            match PEEK.then(|| unsafe { i.class.peek() }).flatten() {
+                Some(cls) => {
+                    if !cls.flags.is_builtin {
+                        let view = std::mem::ManuallyDrop::new(Object::Type(unsafe {
+                            crate::sync::Rc::from_raw(crate::sync::Rc::as_ptr(cls))
+                        }));
+                        visit(&view);
+                    }
+                }
+                None => {
+                    let cls = i.cls();
+                    if !cls.flags.is_builtin {
+                        visit(&Object::Type(cls));
+                    }
+                }
             }
             // A namespace dict that is itself a GC candidate — a
             // `types.ModuleType('foo')` instance's `__dict__`, tracked in
@@ -2099,14 +2482,14 @@ pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) {
                 // Split values: the instance's own children. Their names
                 // are the class's shared strings, which hold nothing (as
                 // CPython's inline values visit only the values).
-                if let Ok(split) = i.dict.split_cell().try_borrow() {
+                with_read::<PEEK, _, _>(i.dict.split_cell(), |split| {
                     for v in split.values() {
                         visit(v);
                     }
-                }
+                });
             } else if let Some(dict) = i.dict.get_shared() {
                 let dict_obj = Object::Dict(dict);
-                if is_tracked(id_of(&dict_obj)) {
+                if is_tracked(&dict_obj) {
                     visit(&dict_obj);
                 } else if let Object::Dict(dict) = &dict_obj {
                     if let Ok(m) = dict.try_borrow() {
@@ -2117,12 +2500,12 @@ pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) {
                     }
                 }
             }
-            if let Ok(slots) = i.slots.try_borrow() {
-                for (k, v) in slots.iter() {
+            with_read::<PEEK, _, _>(&i.slots, |slots| {
+                slots.for_each_entry(|k, v| {
                     visit(&k.0);
                     visit(v);
-                }
-            }
+                });
+            });
             // A built-in *container* subclass (`class C(list)`, `D(dict)`,
             // `S(set)`, …) keeps its payload in `native`; that container is
             // an internal, separately-untracked detail of the instance, so
@@ -2131,7 +2514,7 @@ pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) {
             // prompt reclamation can follow such a chain (a leaf `native`
             // like an `int`/`str` subclass simply has no children).
             if let Some(native) = i.native.get() {
-                traverse_object(native, visit);
+                traverse_impl::<PEEK>(native, visit);
             }
             // A C extension type (RFC 0044) may hold child references in
             // C-managed memory invisible to the dict walk above; give its
@@ -2356,13 +2739,40 @@ fn run_external_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) {
     // The table is append-only and read lock-free (see `HookTable`); a
     // hook may re-enter the collector, and this function, on the same
     // thread.
-    for (matches, traverse) in TRAVERSE_TABLE.iter() {
+    for (matches, traverse) in TRAVERSE_TABLE.iter(hook_kind(obj)) {
         if matches(obj) {
             traverse(obj, visit);
         }
     }
 }
 
+/// [`HookTable`] kinds: a hook names the kinds of object its `matches` can
+/// accept, and only an object of one of them is offered to it.
+pub mod hook_kind {
+    pub const INSTANCE: u8 = 1;
+    pub const GENERATOR: u8 = 2;
+    pub const ITER: u8 = 4;
+    pub const FRAME: u8 = 8;
+    pub const TRACEBACK: u8 = 16;
+    pub const OTHER: u8 = 32;
+    pub const ANY: u8 = u8::MAX;
+}
+
+/// `obj`'s [`hook_kind`].
+#[inline]
+fn hook_kind(obj: &Object) -> u8 {
+    match obj {
+        Object::Instance(_) => hook_kind::INSTANCE,
+        Object::Generator(_) | Object::Coroutine(_) | Object::AsyncGenerator(_) => {
+            hook_kind::GENERATOR
+        }
+        Object::Iter(_) => hook_kind::ITER,
+        Object::Frame(_) => hook_kind::FRAME,
+        Object::Traceback(_) => hook_kind::TRACEBACK,
+        _ => hook_kind::OTHER,
+    }
+}
+
 /// A small append-only registry of `(matches, hook)` function-pointer
 /// pairs with lock-free reads: hooks are registered at interpreter
 /// init and polled on every collection walk, so readers must not take a
@@ -2371,7 +2781,7 @@ const HOOK_TABLE_CAP: usize = 16;
 
 struct HookTable<H: Copy> {
     len: std::sync::atomic::AtomicUsize,
-    slots: [std::sync::OnceLock<(fn(&Object) -> bool, H)>; HOOK_TABLE_CAP],
+    slots: [std::sync::OnceLock<(fn(&Object) -> bool, H, u8)>; HOOK_TABLE_CAP],
 }
 
 impl<H: Copy> HookTable<H> {
@@ -2384,17 +2794,22 @@ impl<H: Copy> HookTable<H> {
         }
     }
 
-    fn push(&self, matches: fn(&Object) -> bool, hook: H) {
+    fn push(&self, kinds: u8, matches: fn(&Object) -> bool, hook: H) {
         let i = self.len.fetch_add(1, Ordering::AcqRel);
         assert!(i < Self::CAP, "too many GC hook registrations");
-        let _ = self.slots[i].set((matches, hook));
+        let _ = self.slots[i].set((matches, hook, kinds));
     }
 
+    /// The hooks offered an object of [`hook_kind`] `kind`.
     #[inline]
-    fn iter(&self) -> impl Iterator<Item = (fn(&Object) -> bool, H)> + '_ {
+    fn iter(&self, kind: u8) -> impl Iterator<Item = (fn(&Object) -> bool, H)> + '_ {
         let n = self.len.load(Ordering::Acquire).min(Self::CAP);
         // A slot past a registration in flight is still unset; skip it.
-        self.slots[..n].iter().filter_map(|s| s.get().copied())
+        self.slots[..n].iter().filter_map(move |s| {
+            s.get()
+                .filter(|(_, _, kinds)| kinds & kind != 0)
+                .map(|&(m, h, _)| (m, h))
+        })
     }
 }
 
@@ -2406,7 +2821,17 @@ pub fn register_traverse(
     matches: fn(&Object) -> bool,
     traverse: fn(&Object, &mut dyn FnMut(&Object)),
 ) {
-    TRAVERSE_TABLE.push(matches, traverse);
+    TRAVERSE_TABLE.push(hook_kind::ANY, matches, traverse);
+}
+
+/// [`register_traverse`] for a hook whose `matches` only accepts objects
+/// of the [`hook_kind`]s in `kinds`.
+pub fn register_traverse_for(
+    kinds: u8,
+    matches: fn(&Object) -> bool,
+    traverse: fn(&Object, &mut dyn FnMut(&Object)),
+) {
+    TRAVERSE_TABLE.push(kinds, matches, traverse);
 }
 
 static CLEAR_TABLE: HookTable<fn(&Object)> = HookTable::new();
@@ -2416,7 +2841,7 @@ static CLEAR_TABLE: HookTable<fn(&Object)> = HookTable::new();
 /// cycles during the collector's clear phase. The companion of
 /// [`register_traverse`] (RFC 0044, WS4).
 fn run_external_clear(obj: &Object) {
-    for (matches, clear) in CLEAR_TABLE.iter() {
+    for (matches, clear) in CLEAR_TABLE.iter(hook_kind(obj)) {
         if matches(obj) {
             clear(obj);
         }
@@ -2427,7 +2852,13 @@ fn run_external_clear(obj: &Object) {
 /// during the collector's clear phase so a matching object can drop the
 /// child references it holds outside the VM's view.
 pub fn register_clear(matches: fn(&Object) -> bool, clear: fn(&Object)) {
-    CLEAR_TABLE.push(matches, clear);
+    CLEAR_TABLE.push(hook_kind::ANY, matches, clear);
+}
+
+/// [`register_clear`] for a hook whose `matches` only accepts objects of
+/// the [`hook_kind`]s in `kinds`.
+pub fn register_clear_for(kinds: u8, matches: fn(&Object) -> bool, clear: fn(&Object)) {
+    CLEAR_TABLE.push(kinds, matches, clear);
 }
 
 /// Drain a container's child references in place. Used during
@@ -2471,7 +2902,9 @@ pub fn clear_object_fields(obj: &Object) -> bool {
             // that dict is still intact.
             run_external_clear(obj);
             if let Ok(mut slots) = i.slots.try_borrow_mut() {
-                *slots = crate::types::SlotStorage::default();
+                if !slots.is_empty_default() {
+                    *slots = crate::types::SlotStorage::default();
+                }
             }
             if i.dict.published().is_none() {
                 let values = i
@@ -2782,30 +3215,27 @@ pub fn collection_in_progress() -> bool {
 /// process-global GC. The inverse of [`track`]; backs the C-API
 /// `PyObject_GC_UnTrack` (RFC 0044, WS4).
 pub fn untrack(obj: &Object) {
-    let id = crate::weakref_registry::id_of(obj);
-    with_state(|s| s.untrack_id(id));
+    with_state(|s| s.untrack(obj));
 }
 
-/// See [`GcState::forget_dead`]; a cheap filter probe first.
+/// Drop the registry entry of the instance at `addr`, which is dying and
+/// recorded registry slot `slot` (see [`GcState::forget_instance`]).
 #[inline]
-pub fn forget_dead(id: ObjectId) {
-    if maybe_tracked(id) {
-        with_state(|s| s.forget_dead(id));
-    }
+pub fn forget_instance(slot: u32, addr: usize) {
+    with_state(|s| s.forget_instance(slot, addr));
 }
 
-/// [`untrack`] by identity.
-pub fn untrack_id(id: ObjectId) {
-    with_state(|s| s.untrack_id(id));
-}
-
-/// [`untrack_id`] for a registered object only: one still in a young set
+/// [`untrack`] for a registered generator only: one still in a young set
 /// stays there. For a finished generator, which holds no frame (and so no
 /// references a cycle could run through): finding it among thousands of
 /// young entries would cost more than leaving it, and a dead entry is
 /// dropped at the next flush.
-pub fn untrack_registered_id(id: ObjectId) {
-    with_state(|s| s.untrack_id_in(id, false));
+pub fn untrack_generator(g: &crate::Rc<crate::object::PyGenerator>) {
+    let slot = g.gc_slot.get();
+    if slot != NO_SLOT {
+        let id = crate::Rc::as_ptr(g) as usize as ObjectId;
+        with_state(|s| s.untrack_slot(slot, id, false));
+    }
 }
 
 /// Reinitialise the process-global cycle collector's locks in a `fork(2)`
@@ -2851,6 +3281,12 @@ pub fn is_atomic(obj: &Object) -> bool {
 /// this is CPython's container-untracking optimization applied at
 /// construction time, and it keeps numeric/string-heavy workloads off
 /// the GC's books entirely.
+/// `WP_GC_STATS`: print each collection's population (read once).
+fn stats_debug() -> bool {
+    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    *ON.get_or_init(|| std::env::var_os("WP_GC_STATS").is_some())
+}
+
 /// `WP_ROOT_DBG`: print each collection's roots (read once).
 fn root_debug() -> bool {
     static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
@@ -3005,26 +3441,16 @@ pub fn maybe_auto_collect() -> bool {
     with_state(GcState::maybe_auto_collect)
 }
 
-/// Convenience: find a tracked handle by object id (O(1) via the
-/// id index, which covers all generations plus the frozen set).
-pub fn find_handle(id: ObjectId) -> Option<HandleRc<TrackedHandle>> {
-    with_state(|s| s.handle_for(id))
-}
-
-/// Convenience: is `id` currently tracked by the cycle GC?
-pub fn is_tracked(id: ObjectId) -> bool {
-    with_state(|s| s.is_tracked(id))
+/// Convenience: is `obj` currently tracked by the cycle GC?
+pub fn is_tracked(obj: &Object) -> bool {
+    with_state(|s| s.is_tracked_obj(obj))
 }
 
-/// Convenience: claim `id`'s finalizer (so a later collection
+/// Convenience: claim `obj`'s finalizer (so a later collection
 /// won't double-run `__del__`). Returns false if it was already
-/// claimed or the object isn't tracked.
-pub fn mark_finalized(id: ObjectId) -> bool {
-    with_state(|s| s.note_finalized(id));
-    match find_handle(id) {
-        Some(h) => !h.finalized.swap(true, Ordering::AcqRel),
-        None => false,
-    }
+/// claimed (a tracked object that isn't claims it).
+pub fn claim_finalizer(obj: &Object) -> bool {
+    with_state(|s| s.claim_finalizer(obj)) != Some(true)
 }
 
 /// Convenience: has `id`'s finalizer already run on the current thread?
@@ -3035,13 +3461,13 @@ pub fn was_finalized(id: ObjectId) -> bool {
 
 /// Convenience: mark `id`'s finalizer as finished on the current thread's GC
 /// (see [`GcState::complete_finalizer`]).
-pub fn complete_finalizer(id: ObjectId) {
-    with_state(|s| s.complete_finalizer(id));
+pub fn complete_finalizer(obj: &Object) {
+    with_state(|s| s.complete_finalizer(obj));
 }
 
 /// Convenience: the live tracked objects with an unrun `__del__` (see
 /// [`GcState::finalization_candidates`]).
-pub fn finalization_candidates() -> Vec<(HandleRc<TrackedHandle>, Object)> {
+pub fn finalization_candidates() -> Vec<Object> {
     with_state(|s| s.finalization_candidates())
 }
 
@@ -3113,34 +3539,49 @@ mod tests {
     use crate::object::DictData;
 
     #[test]
-    fn compact_positions_keep_absent_and_uncached_states_distinct() {
-        for value in [0, 1, (u32::MAX - 2) as usize, usize::MAX] {
-            let hint = CachedSlot::new(value);
-            assert_eq!(hint.load(Ordering::Acquire), value);
-            assert_eq!(hint.swap(3, Ordering::AcqRel), value);
-            assert_eq!(hint.load(Ordering::Acquire), 3);
-            hint.store(value, Ordering::Release);
-            assert_eq!(hint.load(Ordering::Acquire), value);
-        }
-        for value in [(u32::MAX - 1) as usize, usize::MAX - 1] {
-            let hint = CachedSlot::new(value);
-            assert_eq!(hint.load(Ordering::Acquire), usize::MAX - 1);
-            assert_eq!(hint.swap(usize::MAX, Ordering::AcqRel), usize::MAX - 1);
-            assert_eq!(hint.load(Ordering::Acquire), usize::MAX);
-        }
+    fn registry_entries_stay_compact_and_slots_are_reused() {
         #[cfg(target_pointer_width = "64")]
-        {
-            let hint = CachedSlot::new(u32::MAX as usize + 100);
-            assert_eq!(hint.load(Ordering::Acquire), usize::MAX - 1);
-            assert_eq!(std::mem::size_of::<TrackedHandle>(), 32);
+        assert_eq!(std::mem::size_of::<Entry>(), 32);
+        let state = GcState::new();
+        let roots: Vec<_> = (0..3)
+            .map(|_| Object::Dict(Rc::new(RefCell::new(DictData::default()))))
+            .collect();
+        for root in &roots {
+            state.track_now(root);
         }
+        let slab_len = state.reg.borrow().slab.len();
+        state.untrack_id(id_of(&roots[1]));
+        assert!(!state.is_tracked(id_of(&roots[1])));
+        let extra = Object::Dict(Rc::new(RefCell::new(DictData::default())));
+        state.track_now(&extra);
+        // The freed slot is the new entry's.
+        assert_eq!(state.reg.borrow().slab.len(), slab_len);
+        assert!(state.is_tracked(id_of(&extra)));
+        for obj in [&roots[0], &roots[2], &extra] {
+            assert_eq!(placement(&state, obj).map(|p| p.0), Some(0));
+        }
+    }
+
+    /// `obj`'s generation (or [`GEN_FROZEN`]) and position in its list,
+    /// checking that the list holds it there.
+    fn placement(state: &GcState, obj: &Object) -> Option<(u8, u32)> {
+        let reg = state.reg.borrow();
+        let slot = reg.slot_of(obj, &state.tracked_filter)?;
+        let e = reg.entry(slot);
+        let list = if e.gen == GEN_FROZEN {
+            &reg.frozen
+        } else {
+            &reg.gens[usize::from(e.gen)]
+        };
+        assert_eq!(list[e.pos as usize], slot);
+        Some((e.gen, e.pos))
     }
 
     #[test]
-    fn uncacheable_generation_and_frozen_positions_use_identity_fallback() {
+    fn removals_keep_generation_and_frozen_positions() {
         for frozen in [false, true] {
             let state = GcState::new();
-            let roots: Vec<_> = (0..3)
+            let roots: Vec<_> = (0..4)
                 .map(|_| Object::Dict(Rc::new(RefCell::new(DictData::default()))))
                 .collect();
             for root in &roots {
@@ -3149,22 +3590,20 @@ mod tests {
             if frozen {
                 state.freeze_all();
             }
-            let first = state.handle_for(id_of(&roots[0])).unwrap();
-            first.slot.store((u32::MAX - 1) as usize, Ordering::Release);
-            state.untrack_id(first.id);
-            assert!(!state.is_tracked(first.id));
-            let moved = state.handle_for(id_of(&roots[2])).unwrap();
-            assert_eq!(moved.slot.load(Ordering::Acquire), 0);
-            let second = state.handle_for(id_of(&roots[1])).unwrap();
-            // A cacheable but stale position must also validate identity.
-            second.slot.store(0, Ordering::Release);
-            state.untrack_id(second.id);
-            assert!(!state.is_tracked(second.id));
-            assert!(state.is_tracked(moved.id));
-            assert_eq!(state.freeze_count(), usize::from(frozen));
-            state.untrack_id(moved.id);
-            assert!(!state.is_tracked(moved.id));
+            state.untrack_id(id_of(&roots[0]));
+            assert!(!state.is_tracked(id_of(&roots[0])));
+            // The last member moved into the vacated position.
+            assert_eq!(placement(&state, &roots[3]).unwrap().1, 0);
+            state.untrack_id(id_of(&roots[2]));
+            for root in [&roots[1], &roots[3]] {
+                let (gen, _) = placement(&state, root).unwrap();
+                assert_eq!(gen == GEN_FROZEN, frozen);
+            }
+            assert_eq!(state.freeze_count(), if frozen { 2 } else { 0 });
+            state.untrack_id(id_of(&roots[1]));
+            state.untrack_id(id_of(&roots[3]));
             assert_eq!(state.freeze_count(), 0);
+            assert_eq!(state.population(), 0);
         }
     }
 
@@ -3199,44 +3638,27 @@ mod tests {
         for (collection, expected) in [(0, 1), (1, 2), (2, 2), (2, 2)] {
             assert_eq!(state.collect(collection), 0);
             for (slot, root) in roots.iter().enumerate() {
-                let handle = state.handle_for(id_of(root)).unwrap();
-                assert_eq!(handle.generation.load(Ordering::Acquire), expected);
-                assert_eq!(handle.color.load(Ordering::Acquire), color::White);
-                assert_eq!(handle.slot.load(Ordering::Acquire), slot);
+                assert_eq!(placement(&state, root), Some((expected, slot as u32)));
             }
         }
         state.freeze_all();
         assert_eq!(state.freeze_count(), roots.len());
         assert_eq!(state.collect(2), 0);
         for root in &roots {
-            let handle = state.handle_for(id_of(root)).unwrap();
-            assert_eq!(handle.color.load(Ordering::Acquire), color::Frozen);
+            assert_eq!(placement(&state, root).unwrap().0, GEN_FROZEN);
         }
         state.untrack_id(id_of(&roots[1]));
         assert!(!state.is_tracked(id_of(&roots[1])));
         assert_eq!(state.freeze_count(), 3);
-        let moved = state.handle_for(id_of(&roots[3])).unwrap();
-        assert_eq!(moved.slot.load(Ordering::Acquire), 1);
+        assert_eq!(placement(&state, &roots[3]).unwrap().1, 1);
         state.unfreeze_all();
         assert_eq!(state.freeze_count(), 0);
         for (slot, index) in [0, 3, 2].into_iter().enumerate() {
-            let handle = state.handle_for(id_of(&roots[index])).unwrap();
-            assert_eq!(handle.generation.load(Ordering::Acquire), 0);
-            assert_eq!(handle.color.load(Ordering::Acquire), color::White);
-            assert_eq!(handle.slot.load(Ordering::Acquire), slot);
+            assert_eq!(placement(&state, &roots[index]), Some((0, slot as u32)));
         }
         state.untrack_id(id_of(&roots[3]));
         assert!(!state.is_tracked(id_of(&roots[3])));
-        let moved = state.handle_for(id_of(&roots[2])).unwrap();
-        assert_eq!(moved.slot.load(Ordering::Acquire), 1);
-    }
-
-    #[test]
-    fn generation_constructor_rejects_out_of_range_indices() {
-        for generation in [N_GENERATIONS, u8::MAX as usize + 1, usize::MAX] {
-            let result = std::panic::catch_unwind(|| TrackedHandle::new(&Object::None, generation));
-            assert!(result.is_err());
-        }
+        assert_eq!(placement(&state, &roots[2]).unwrap().1, 1);
     }
 
     #[test]
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ba40f79e..e7527cb5 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -1143,7 +1143,8 @@ impl Default for Interpreter {
         // frames (their `Frame` type is private to this module).
         static GEN_TRAVERSE: std::sync::Once = std::sync::Once::new();
         GEN_TRAVERSE.call_once(|| {
-            crate::gc_trace::register_traverse(
+            crate::gc_trace::register_traverse_for(
+                crate::gc_trace::hook_kind::GENERATOR,
                 |o| {
                     matches!(
                         o,
@@ -1157,17 +1158,23 @@ impl Default for Interpreter {
             // live set it walks. Teach the cycle GC to trace through it so
             // e.g. `obj.x = iter(set_containing_obj)` is collectable
             // (`test_set.test_container_iterator`).
-            crate::gc_trace::register_traverse(|o| matches!(o, Object::Iter(_)), iter_traverse);
+            crate::gc_trace::register_traverse_for(
+                crate::gc_trace::hook_kind::ITER,
+                |o| matches!(o, Object::Iter(_)),
+                iter_traverse,
+            );
             // A dying exception's `__traceback__` is often the sole owner of
             // the unwound frames — and their locals — below it; CPython's
             // refcounting frees that whole chain the instant the exception
             // dies. Teach the prompt-reap cascade (and anything else walking
             // `traverse_object`) to follow traceback → frame → locals.
-            crate::gc_trace::register_traverse(
+            crate::gc_trace::register_traverse_for(
+                crate::gc_trace::hook_kind::TRACEBACK,
                 |o| matches!(o, Object::Traceback(_)),
                 py_traceback_traverse,
             );
-            crate::gc_trace::register_traverse(
+            crate::gc_trace::register_traverse_for(
+                crate::gc_trace::hook_kind::FRAME,
                 |o| matches!(o, Object::Frame(_)),
                 py_frame_traverse,
             );
@@ -2755,7 +2762,7 @@ impl Interpreter {
                 // finalized and clear its deferral flag so the next collection
                 // treats it as plain garbage (if it wasn't resurrected) and
                 // never runs `__del__` a second time.
-                crate::gc_trace::complete_finalizer(crate::weakref_registry::id_of(&obj));
+                crate::gc_trace::complete_finalizer(&obj);
             }
             // Weakref callbacks run after finalizers (CPython's order);
             // errors route through the unraisable hook. WeavePy reaps at
@@ -3066,29 +3073,22 @@ impl Interpreter {
             Some(Object::Module(m)) => m.dict.borrow().iter().map(|(_k, v)| v.clone()).collect(),
             _ => Vec::new(),
         };
-        let mut sys_deferred: Vec<(crate::sync::Rc<crate::gc_trace::TrackedHandle>, Object)> =
-            Vec::new();
+        let mut sys_deferred: Vec<Object> = Vec::new();
         for _ in 0..8 {
             let candidates = crate::gc_trace::finalization_candidates();
             if candidates.is_empty() {
                 break;
             }
-            for (handle, obj) in candidates {
+            for obj in candidates {
                 if sys_values.iter().any(|v| v.is_same(&obj)) {
-                    if !sys_deferred
-                        .iter()
-                        .any(|(h, _)| crate::sync::Rc::ptr_eq(h, &handle))
-                    {
-                        sys_deferred.push((handle, obj));
+                    if !sys_deferred.iter().any(|o| o.is_same(&obj)) {
+                        sys_deferred.push(obj);
                     }
                     continue;
                 }
-                // `swap` claims the finalizer so the cycle collector
-                // and a later shutdown pass can't double-run it.
-                if handle
-                    .finalized
-                    .swap(true, std::sync::atomic::Ordering::AcqRel)
-                {
+                // Claiming the finalizer means the cycle collector and a
+                // later shutdown pass can't double-run it.
+                if !crate::gc_trace::claim_finalizer(&obj) {
                     continue;
                 }
                 self.invoke_finalizer(&obj);
@@ -3098,11 +3098,8 @@ impl Interpreter {
         }
         if !sys_deferred.is_empty() {
             crate::vm_singletons::set_unraisable_silenced(true);
-            for (handle, obj) in sys_deferred {
-                if handle
-                    .finalized
-                    .swap(true, std::sync::atomic::Ordering::AcqRel)
-                {
+            for obj in sys_deferred {
+                if !crate::gc_trace::claim_finalizer(&obj) {
                     continue;
                 }
                 self.invoke_finalizer(&obj);
@@ -19348,7 +19345,7 @@ impl Interpreter {
                                 let id = Rc::as_ptr(instance) as usize as u64;
                                 eprintln!("First predicate instance drop miss: owners={}, deferred={}, maybe_tracked={}, tracked={}, weakrefs={}",
                                     Rc::strong_count(instance), instance.is_gc_deferred(),
-                                    gc_trace::maybe_tracked(id), gc_trace::is_tracked(id),
+                                    gc_trace::maybe_tracked(id), gc_trace::is_tracked(operand),
                                     crate::weakref_registry::may_have_weakrefs(id));
                             }
                         }
@@ -43344,17 +43341,16 @@ impl Interpreter {
         if gen.kind == crate::object::CoroutineKind::AsyncGenerator {
             return;
         }
-        // A generator still in the collector's young set has no index
+        // A generator still in the collector's young set has no registry
         // entry to remove (the set drops it once it's dead).
-        if !gen.gc_registered.get() {
+        if !gen.gc_slot.is_registered() {
             return;
         }
         let id = Rc::as_ptr(gen) as usize as u64;
         if crate::weakref_registry::count_for(id) > 0 {
             return;
         }
-        gen.gc_registered.set(false);
-        gc_trace::untrack_registered_id(id);
+        gc_trace::untrack_generator(gen);
     }
 
     fn park_suspended_boxed(gen: &Rc<PyGenerator>, boxed: Box<Frame>) {
@@ -51997,6 +51993,7 @@ impl Interpreter {
             slot_seed: RefCell::new(None),
             closure_cells: std::sync::OnceLock::new(),
             defaults_override: crate::object::OverrideFlag::new(false),
+            gc_slot: crate::gc_trace::GcSlot::new(),
         })))
     }
 
@@ -73623,6 +73620,7 @@ fn new_function(
         attrs: RefCell::new(None),
         slots_raw: crate::sync::OnceBox::new(),
         slot_seed: RefCell::new(Some(crate::object::LazySlots(module))),
+        gc_slot: crate::gc_trace::GcSlot::new(),
         closure_cells: std::sync::OnceLock::new(),
         defaults_override: crate::object::OverrideFlag::new(false),
     };
@@ -77167,7 +77165,7 @@ assert loop(2000) == 1999000
                 "  function owners={}, maybe_tracked={}, tracked={}",
                 Rc::strong_count(function),
                 gc_trace::maybe_tracked(id),
-                gc_trace::is_tracked(id)
+                gc_trace::is_tracked(&Object::Function(function.clone()))
             );
             if *name == "default_predicate" {
                 if let Some(receiver @ Object::Instance(inst)) = function.defaults.first() {
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 3c21ca35..76d0d507 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -4919,6 +4919,9 @@ pub struct PyFunction {
     /// slot-store probe for the overwhelmingly common function that never
     /// had them overridden.
     pub defaults_override: OverrideFlag,
+    /// The function's slot in the cycle collector's registry (see
+    /// [`crate::gc_trace::GcSlot`]).
+    pub gc_slot: crate::gc_trace::GcSlot,
 }
 
 /// A function's getset slots before their first read (see
@@ -5368,9 +5371,10 @@ pub struct PyGenerator {
     /// finalizer (e.g. a PEP 525 hook that declined to close it) is
     /// not resurrected and re-finalized forever on the next drop.
     pub finalize_ran: crate::sync::Cell<bool>,
-    /// Whether the cycle collector's index holds an entry for this
-    /// generator (one still in a young set has none).
-    pub gc_registered: crate::sync::Cell<bool>,
+    /// The generator's slot in the cycle collector's registry (see
+    /// [`crate::gc_trace::GcSlot`]); unregistered while it is still in a
+    /// young set.
+    pub gc_slot: crate::gc_trace::GcSlot,
 }
 
 /// The state of a [`PyGenerator`] that few carry.
@@ -5511,7 +5515,7 @@ impl PyGenerator {
             extras: RefCell::new(None),
             hooks_inited: crate::sync::Cell::new(false),
             finalize_ran: crate::sync::Cell::new(false),
-            gc_registered: crate::sync::Cell::new(false),
+            gc_slot: crate::gc_trace::GcSlot::new(),
         }
     }
 
diff --git a/crates/weavepy-vm/src/rc.rs b/crates/weavepy-vm/src/rc.rs
index 72ab4bf7..23c8ac26 100644
--- a/crates/weavepy-vm/src/rc.rs
+++ b/crates/weavepy-vm/src/rc.rs
@@ -712,9 +712,35 @@ impl<T: 'static> Weak<T> {
 impl<T: ?Sized + 'static> Weak<T> {
     #[inline]
     pub fn upgrade(&self) -> Option<Rc<T>> {
+        if refcounts_biased() {
+            // `Weak::upgrade`'s compare-and-swap loop guards against another
+            // thread's release racing the increment, which the bias rules
+            // out. (A dangling handle reads a count of zero.)
+            if self.0.strong_count() == 0 {
+                return None;
+            }
+            let p = self.0.as_ptr();
+            // SAFETY: the payload is live (its strong count is not zero),
+            // and the new owner accounts for the added reference.
+            unsafe {
+                increment_strong(p);
+                return Some(Self::from_raw_rc(p));
+            }
+        }
         self.0.upgrade().map(Rc::from_arc)
     }
 
+    /// The [`Rc`] owning the strong reference the caller added for `p`.
+    ///
+    /// # Safety
+    ///
+    /// As [`Rc::from_raw`].
+    #[inline(always)]
+    unsafe fn from_raw_rc(p: *const T) -> Rc<T> {
+        // SAFETY: forwarded to the caller.
+        unsafe { Rc::from_raw(p) }
+    }
+
     pub fn strong_count(&self) -> usize {
         self.0.strong_count()
     }
diff --git a/crates/weavepy-vm/src/shared_value.rs b/crates/weavepy-vm/src/shared_value.rs
index 63b19c42..64157dc2 100644
--- a/crates/weavepy-vm/src/shared_value.rs
+++ b/crates/weavepy-vm/src/shared_value.rs
@@ -245,6 +245,19 @@ impl<T: ?Sized + ThinPayload> ThinWeak<T> {
     pub fn upgrade(&self) -> Option<ThinArc<T>> {
         self.weak_view().upgrade().map(ThinArc::from_arc)
     }
+
+    /// A strong handle on the payload that owns no reference of its own.
+    ///
+    /// # Safety
+    ///
+    /// The payload must be alive, and stay so while the view is used; the
+    /// view must never be dropped (it owns no reference to release).
+    pub(crate) unsafe fn borrowed_arc(&self) -> ManuallyDrop<ThinArc<T>> {
+        ManuallyDrop::new(ThinArc {
+            data: self.data,
+            ownership: PhantomData,
+        })
+    }
     pub fn strong_count(&self) -> usize {
         self.weak_view().strong_count()
     }
diff --git a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
index 204a9e06..b54f5b90 100644
--- a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
@@ -729,8 +729,9 @@ fn fut_gc_clear(obj: &Object) {
 fn register_gc_hooks() {
     static ONCE: std::sync::Once = std::sync::Once::new();
     ONCE.call_once(|| {
-        crate::gc_trace::register_traverse(fut_gc_matches, fut_gc_traverse);
-        crate::gc_trace::register_clear(fut_gc_matches, fut_gc_clear);
+        let kind = crate::gc_trace::hook_kind::INSTANCE;
+        crate::gc_trace::register_traverse_for(kind, fut_gc_matches, fut_gc_traverse);
+        crate::gc_trace::register_clear_for(kind, fut_gc_matches, fut_gc_clear);
     });
 }
 
diff --git a/crates/weavepy-vm/src/stdlib/gc_real.rs b/crates/weavepy-vm/src/stdlib/gc_real.rs
index 67e47ec0..e2ddc654 100644
--- a/crates/weavepy-vm/src/stdlib/gc_real.rs
+++ b/crates/weavepy-vm/src/stdlib/gc_real.rs
@@ -457,7 +457,7 @@ fn find_paths_dbg(args: &[Object]) -> Result<Object, RuntimeError> {
                     out.push(Object::from_str(p.join(" -> ")));
                     continue;
                 }
-                if gc_trace::is_tracked(cid) || !seen.insert(cid) {
+                if gc_trace::is_tracked(&child) || !seen.insert(cid) {
                     continue;
                 }
                 let mut p = path.clone();
@@ -504,28 +504,22 @@ fn object_is_tracked(target: &Object) -> bool {
         // answers as CPython's (always tracked) and is tracked from here on.
         Object::Instance(inst) => {
             inst.ensure_gc_tracked();
-            gc_trace::with_state(|s| s.is_tracked_instance(id_of(target)))
+            gc_trace::with_state(|s| s.is_tracked_instance(target))
         }
         // A function, generator or coroutine may still be young.
         Object::Function(_) | Object::Generator(_) | Object::Coroutine(_) => {
-            gc_trace::with_state(|s| s.is_tracked_instance(id_of(target)))
+            gc_trace::with_state(|s| s.is_tracked_instance(target))
         }
         // A list/dict/set born holding only scalars has its tracking
         // deferred; CPython tracks it from birth, so answer as CPython
         // does and hand it to the collector from here on.
-        Object::List(_) | Object::Dict(_) | Object::Set(_) => {
-            let id = id_of(target);
-            gc_trace::with_state(|s| {
-                if !s.is_tracked_instance(id) {
-                    s.track_now(target);
-                }
-                s.is_tracked_instance(id)
-            })
-        }
-        other => {
-            let id = id_of(other);
-            gc_trace::with_state(|s| s.is_tracked(id))
-        }
+        Object::List(_) | Object::Dict(_) | Object::Set(_) => gc_trace::with_state(|s| {
+            if !s.is_tracked_instance(target) {
+                s.track_now(target);
+            }
+            s.is_tracked_instance(target)
+        }),
+        other => gc_trace::with_state(|s| s.is_tracked_obj(other)),
     }
 }
 
@@ -596,8 +590,7 @@ fn track_obj(args: &[Object]) -> Result<Object, RuntimeError> {
 
 fn untrack_obj(args: &[Object]) -> Result<Object, RuntimeError> {
     if let Some(o) = args.first() {
-        let id = id_of(o);
-        gc_trace::with_state(|s| s.untrack_id(id));
+        gc_trace::untrack(o);
     }
     Ok(Object::None)
 }
diff --git a/crates/weavepy-vm/src/stdlib/multiprocessing_mod.rs b/crates/weavepy-vm/src/stdlib/multiprocessing_mod.rs
index c240c91a..3c5744b4 100644
--- a/crates/weavepy-vm/src/stdlib/multiprocessing_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/multiprocessing_mod.rs
@@ -599,6 +599,7 @@ fn make_semlock_instance(inner: Arc<SemInner>) -> Object {
         finalize_ran: crate::sync::Cell::new(false),
         deferred: crate::sync::Cell::new(false),
         c_body: crate::types::CBody::default(),
+        gc_slot: crate::gc_trace::GcSlot::new(),
     });
     Object::Instance(inst)
 }
@@ -1232,6 +1233,7 @@ fn nt_make_semlock_instance(inner: &Arc<NtSemInner>) -> Object {
         finalize_ran: crate::sync::Cell::new(false),
         deferred: crate::sync::Cell::new(false),
         c_body: crate::types::CBody::default(),
+        gc_slot: crate::gc_trace::GcSlot::new(),
     });
     Object::Instance(inst)
 }
diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs
index 0b94f6b2..f596363b 100644
--- a/crates/weavepy-vm/src/stdlib/thread_real.rs
+++ b/crates/weavepy-vm/src/stdlib/thread_real.rs
@@ -731,6 +731,7 @@ fn make_lock_object(lock: Arc<RealLock>) -> Object {
         finalize_ran: crate::sync::Cell::new(false),
         deferred: crate::sync::Cell::new(false),
         c_body: crate::types::CBody::default(),
+        gc_slot: crate::gc_trace::GcSlot::new(),
     });
     Object::Instance(inst)
 }
@@ -916,6 +917,7 @@ fn make_rlock_object(rlock: Arc<RealRLock>) -> Object {
         finalize_ran: crate::sync::Cell::new(false),
         deferred: crate::sync::Cell::new(false),
         c_body: crate::types::CBody::default(),
+        gc_slot: crate::gc_trace::GcSlot::new(),
     });
     Object::Instance(inst)
 }
@@ -1834,6 +1836,7 @@ fn make_thread_handle_object(state: Arc<ThreadHandleState>, ident: Object) -> Ob
         finalize_ran: crate::sync::Cell::new(false),
         deferred: crate::sync::Cell::new(false),
         c_body: crate::types::CBody::default(),
+        gc_slot: crate::gc_trace::GcSlot::new(),
     });
     Object::Instance(inst)
 }
diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs
index f7f5ef48..d83006fb 100644
--- a/crates/weavepy-vm/src/stdlib/weakref_real.rs
+++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs
@@ -1189,6 +1189,7 @@ fn make_ref_object_with_class(
         finalize_ran: crate::sync::Cell::new(false),
         deferred: crate::sync::Cell::new(false),
         c_body: crate::types::CBody::default(),
+        gc_slot: crate::gc_trace::GcSlot::new(),
     });
     // Back-pointer so `obj.__weakref__` / `getweakrefs(obj)` can return
     // this same wrapper object.
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index bad33144..72cf11ae 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -798,6 +798,11 @@ pub struct TypeObject {
     /// Every exception construction asked the MRO for nine names. Last
     /// field: the hot ones above keep their offsets.
     pub exc_families: Cell<u64>,
+    /// The class's slot in the cycle collector's registry, or
+    /// [`crate::gc_trace::NO_SLOT`] while it isn't registered (as
+    /// [`PyInstance::gc_slot`]: every instance's traversal visits its
+    /// class, so finding the class's entry must not cost a hash probe).
+    pub gc_slot: crate::gc_trace::GcSlot,
 }
 
 /// A dying TypeObject clears the weak references watching it.
@@ -1144,6 +1149,7 @@ impl TypeObject {
         }
         let ty = Rc::new(TypeObject {
             name: name.to_owned(),
+            gc_slot: crate::gc_trace::GcSlot::new(),
             qualname: RefCell::new(qualname),
             bases: RefCell::new(bases.clone()),
             mro: RefCell::new(Vec::new()),
@@ -2957,6 +2963,44 @@ impl SlotStorage {
             // (An unset laid-out slot.)
             .filter(|(_, value)| !matches!(value, Object::Unbound))
     }
+
+    /// Call `f` on each populated slot, as [`Self::iter`] yields them, by
+    /// a direct walk of the representation (the collector's traversal).
+    #[inline]
+    pub(crate) fn for_each_entry(&self, mut f: impl FnMut(&DictKey, &Object)) {
+        let mut each = |k: &DictKey, v: &Object| {
+            if !matches!(v, Object::Unbound) {
+                f(k, v);
+            }
+        };
+        match &self.data {
+            SlotData::Single { key, value } => {
+                if let Some(key) = key {
+                    each(key, value);
+                }
+            }
+            SlotData::Small(entries) => {
+                for (k, v) in entries {
+                    each(k, v);
+                }
+            }
+            SlotData::Many(table) => {
+                for (k, v) in table.iter() {
+                    each(k, v);
+                }
+            }
+            SlotData::Fixed { layout, values } => {
+                for (k, v) in layout.iter().zip(values.iter()) {
+                    each(k, v);
+                }
+            }
+            SlotData::Packed(p) => {
+                for (k, v) in p.layout().iter().zip(p.values().iter()) {
+                    each(k, v);
+                }
+            }
+        }
+    }
 }
 
 /// Slot storage for targets where an inline object pair would enlarge
@@ -3023,6 +3067,14 @@ impl SlotStorage {
     pub fn iter(&self) -> impl Iterator<Item = (&DictKey, &Object)> {
         self.0.iter()
     }
+
+    /// Call `f` on each populated slot.
+    #[inline]
+    pub(crate) fn for_each_entry(&self, mut f: impl FnMut(&DictKey, &Object)) {
+        for (k, v) in self.0.iter() {
+            f(k, v);
+        }
+    }
 }
 
 /// An instance of a user-defined class.
@@ -3095,6 +3147,12 @@ pub struct PyInstance {
     /// mutation reached through the dict alone (`vars(obj)['x'] = y`)
     /// still starts tracking; the two are set and cleared together.
     pub deferred: Cell<bool>,
+    /// The instance's slot in the cycle collector's registry, or
+    /// [`crate::gc_trace::NO_SLOT`] while it isn't registered. The
+    /// registry keeps no id index for instances: this field is how a
+    /// collection finds an instance's entry, and how the instance's
+    /// `Drop` removes it.
+    pub gc_slot: crate::gc_trace::GcSlot,
     /// The stable C "inline body" this instance owns once it has crossed
     /// into a C extension that reads its fields at fixed `tp_basicsize`
     /// offsets (RFC 0045, wave 3). `0` for the overwhelmingly common case
@@ -3126,6 +3184,7 @@ impl PyInstance {
             finalize_ran: Cell::new(false),
             deferred: Cell::new(false),
             c_body: CBody::default(),
+            gc_slot: crate::gc_trace::GcSlot::new(),
         }
     }
 
@@ -3143,6 +3202,7 @@ impl PyInstance {
             finalize_ran: Cell::new(false),
             deferred: Cell::new(false),
             c_body: CBody::default(),
+            gc_slot: crate::gc_trace::GcSlot::new(),
         }
     }
 
@@ -3495,7 +3555,10 @@ impl Drop for PyInstance {
         crate::weakref_registry::on_death(id);
         // A collector registry entry's weak handle would keep this
         // allocation until its generation is next collected.
-        crate::gc_trace::forget_dead(id);
+        let slot = self.gc_slot.get();
+        if slot != crate::gc_trace::NO_SLOT {
+            crate::gc_trace::forget_instance(slot, id as usize);
+        }
     }
 }
 
diff --git a/crates/weavepy-vm/src/weak_object.rs b/crates/weavepy-vm/src/weak_object.rs
index 65cefc96..56755ffa 100644
--- a/crates/weavepy-vm/src/weak_object.rs
+++ b/crates/weavepy-vm/src/weak_object.rs
@@ -54,6 +54,42 @@ macro_rules! weak_object {
             pub fn is_dead(&self) -> bool {
                 self.strong_count() == 0
             }
+
+            /// Run `f` on the object, while it is alive, through a handle
+            /// that takes no reference (`None` once it died).
+            ///
+            /// # Safety
+            ///
+            /// Nothing `f` does may release the object's last reference.
+            #[inline]
+            pub unsafe fn with_borrowed<R>(&self, f: impl FnOnce(&Object) -> R) -> Option<R> {
+                if self.is_dead() {
+                    return None;
+                }
+                // SAFETY: the payload is alive (its strong count isn't
+                // zero), stays so per the caller, and the view is never
+                // dropped, so it releases nothing.
+                let obj = std::mem::ManuallyDrop::new(unsafe {
+                    match self {
+                        Self::Tuple(w) => {
+                            Object::Tuple(std::mem::ManuallyDrop::into_inner(w.borrowed_arc()))
+                        }
+                        $(Self::$variant(w) => Object::$variant(Rc::from_raw(w.as_ptr())),)*
+                    }
+                });
+                Some(f(&obj))
+            }
+
+            /// The payload's address, as [`Object::payload_addr`] reports
+            /// it for a strong owner (valid after the object died: the
+            /// handle keeps the allocation).
+            #[inline]
+            pub fn addr(&self) -> usize {
+                match self {
+                    Self::Tuple(w) => w.addr(),
+                    $(Self::$variant(w) => w.as_ptr().cast::<()>() as usize,)*
+                }
+            }
         }
     };
     (@payload Long) => { Weak<num_bigint::BigInt> };
diff --git a/tests/regrtest/test_gc_instance_registry.py b/tests/regrtest/test_gc_instance_registry.py
new file mode 100644
index 00000000..9cc0e044
--- /dev/null
+++ b/tests/regrtest/test_gc_instance_registry.py
@@ -0,0 +1,147 @@
+"""Instances that record their own collector slot: registration, death, and reuse."""
+import gc
+import weakref
+
+previous_enabled = gc.isenabled()
+gc.disable()
+gc.collect()
+
+
+class Plain:
+    pass
+
+
+class Node:
+    def __init__(self, tag):
+        self.tag = tag
+        self.other = None
+
+
+finalized = []
+
+
+class Finalized:
+    def __init__(self, tag):
+        self.tag = tag
+        self.me = self
+
+    def __del__(self):
+        finalized.append(self.tag)
+
+
+# A plain instance answers as tracked, and keeps answering through churn
+# that frees and reuses the slots around it.
+keep = Plain()
+assert gc.is_tracked(keep)
+for _ in range(3):
+    churn = [Node(i) for i in range(500)]
+    for node in churn:
+        node.other = churn
+    del churn
+    assert gc.is_tracked(keep)
+assert gc.collect() >= 500
+assert any(o is keep for o in gc.get_objects())
+
+
+# Instances that die by reference count leave the registry at once: a
+# collection afterwards finds nothing.
+def make_and_drop(n):
+    for i in range(n):
+        a = Node(i)
+        a.other = [a.tag]
+        del a
+
+
+make_and_drop(1000)
+assert gc.collect() == 0
+
+
+# Self-cycles are reclaimed, counted, and their weakref callbacks run.
+callbacks = []
+refs = []
+for i in range(200):
+    n = Node(i)
+    n.other = n
+    refs.append(weakref.ref(n, lambda r: callbacks.append(1)))
+del n
+collected = gc.collect()
+assert collected >= 200, collected
+assert all(r() is None for r in refs)
+assert len(callbacks) == 200, len(callbacks)
+
+
+# Finalizers run once; a cycle whose __del__ ran is reclaimed by a later
+# collection, and the count only covers objects actually reclaimed.
+for i in range(50):
+    Finalized(i)
+gc.collect()
+assert sorted(finalized) == list(range(50)), finalized
+assert gc.collect() == 0
+assert sorted(finalized) == list(range(50))
+
+
+# Resurrection: a finalizer that stores its object keeps it alive and
+# registered, and never runs again.
+saved = []
+
+
+class Phoenix:
+    def __init__(self):
+        self.me = self
+
+    def __del__(self):
+        saved.append(self)
+
+
+Phoenix()
+gc.collect()
+assert len(saved) == 1
+phoenix = saved.pop()
+assert gc.is_tracked(phoenix)
+del phoenix
+gc.collect()
+assert saved == []
+
+
+# An older instance that dies while a younger cycle is being cleared.
+class Holder:
+    pass
+
+
+old = Holder()
+gc.collect()
+gc.collect()
+young = Node("young")
+young.other = young
+young.tag = old
+del old
+del young
+assert gc.collect() >= 1
+
+
+# Frozen instances survive collections and rejoin generation 0 on unfreeze,
+# including ones that died while frozen.
+frozen_cycle = Node("frozen")
+frozen_cycle.other = frozen_cycle
+transient = Node("transient")
+wr = weakref.ref(frozen_cycle)
+gc.freeze()
+del transient
+del frozen_cycle
+gc.collect()
+assert wr() is not None
+gc.unfreeze()
+gc.collect()
+assert wr() is None
+
+
+# gc.get_referrers sees an instance holding a list.
+target = []
+holder = Node("holder")
+holder.other = target
+assert any(r is holder or (isinstance(r, dict) and r.get("other") is target)
+           for r in gc.get_referrers(target))
+
+if previous_enabled:
+    gc.enable()
+print("instance registry: ok")

From ff44d8ae7b9a65c7f84b42f3031427a70ea12c80 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 01:20:19 -0700
Subject: [PATCH 030/173] perf: mark reachability along recorded edges, not a
 second walk

A collection walked every candidate twice: once to subtract internal
references, and again from the externally referenced candidates to
mark what they reach, re-running every traverse hook (a coroutine's
frame, an asyncio future's state) and every child lookup. The first
walk now records each candidate's edges to other candidates (4 bytes
an edge, in a flat array), and marking follows those.

Also: the child lookup caches every answer (registry candidate,
temporary, or neither) by address, so the children reached from many
parents (module namespaces, shared functions, deferred instances) cost
one probe; the asyncio future traverse hook is offered only instances
with slot storage and finds the future's state once; and a flush
registers the young sets under one registry borrow, moving each weak
handle into its entry instead of upgrading and re-downgrading it.

Instruction counts per iteration (macOS, PYTHONHASHSEED=0):
  gc.collect() over 2N live objects   25.5k -> 19.0k
  instance holding a list, kept        8.50k -> 7.40k
  self-cyclic instance churn           3.39k -> 3.27k
Suite vs the previous commit: async_tree 0.930, comprehensions 0.976,
geomean 0.9975 (against m1: async_tree 0.912, comprehensions 0.942,
xml_etree 0.974, geomean 0.9946).
---
 crates/weavepy-vm/src/gc_trace.rs           | 511 +++++++++++++-------
 crates/weavepy-vm/src/stdlib/asyncio_mod.rs |  15 +-
 2 files changed, 337 insertions(+), 189 deletions(-)

diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs
index 935c6009..e33f55b6 100644
--- a/crates/weavepy-vm/src/gc_trace.rs
+++ b/crates/weavepy-vm/src/gc_trace.rs
@@ -205,6 +205,9 @@ struct Entry {
     /// While [`F_CAND`] is set: the object's references from outside the
     /// candidates, as the collection counts them (CPython's `gc_refs`).
     gc_refs: std::cell::Cell<i32>,
+    /// While [`F_CAND`] is set: the candidate's node number in the
+    /// collection's edge lists.
+    node: std::cell::Cell<u32>,
     /// Generation (0..N_GENERATIONS), [`GEN_FROZEN`] or [`GEN_FREE`].
     gen: u8,
     /// [`F_FINALIZED`], [`F_QUEUED`] and [`F_CAND`].
@@ -252,6 +255,27 @@ fn intrusive_slot(obj: &Object) -> Option<&GcSlot> {
     }
 }
 
+/// The [`GcSlot`] of the object `w` names, of a kind that records its own.
+///
+/// # Safety
+///
+/// The object must be alive.
+#[inline]
+unsafe fn weak_intrusive_slot(w: &WeakObject) -> Option<&GcSlot> {
+    // SAFETY: alive, per the caller; the handle keeps the allocation.
+    unsafe {
+        match w {
+            WeakObject::Instance(w) => Some(&(*w.as_ptr()).gc_slot),
+            WeakObject::Type(w) => Some(&(*w.as_ptr()).gc_slot),
+            WeakObject::Function(w) => Some(&(*w.as_ptr()).gc_slot),
+            WeakObject::Generator(w) | WeakObject::Coroutine(w) | WeakObject::AsyncGenerator(w) => {
+                Some(&(*w.as_ptr()).gc_slot)
+            }
+            _ => None,
+        }
+    }
+}
+
 /// Forget the registry slot a live object records (its entry is going).
 /// A dead one's field is gone with it.
 #[inline]
@@ -341,6 +365,7 @@ impl Registry {
             obj: Some(obj),
             pos,
             gc_refs: std::cell::Cell::new(0),
+            node: std::cell::Cell::new(0),
             gen,
             flags: std::cell::Cell::new(flags),
         };
@@ -791,29 +816,9 @@ impl GcState {
                 }
             }
         }
-        if let Ok(mut lists) = self.young_lists.try_borrow_mut() {
-            for w in std::mem::take(&mut *lists) {
-                if let Some(l) = w.upgrade() {
-                    self.register(&Object::List(l), false);
-                }
-            }
-        }
-        if let Ok(mut dicts) = self.young_dicts.try_borrow_mut() {
-            for w in std::mem::take(&mut *dicts) {
-                if let Some(d) = w.upgrade() {
-                    self.register(&Object::Dict(d), false);
-                }
-            }
-        }
-        let young = match self.young.try_borrow_mut() {
-            Ok(mut young) if !young.is_empty() => std::mem::take(&mut *young),
-            _ => return,
-        };
-        for w in young {
-            if let Some(inst) = w.upgrade() {
-                self.register(&Object::Instance(inst), false);
-            }
-        }
+        self.flush_set(&self.young_lists, WeakObject::List);
+        self.flush_set(&self.young_dicts, WeakObject::Dict);
+        self.flush_set(&self.young, WeakObject::Instance);
     }
 
     /// The young instances and functions still alive (the dead ones are
@@ -986,10 +991,6 @@ impl GcState {
     /// may be a finalized one's, recycled; a young instance's own
     /// finalizer may already have run, and its entry says so.
     fn register(&self, obj: &Object, fresh: bool) -> bool {
-        let Some(key) = obj.payload_addr() else {
-            return false;
-        };
-        let key = key as ObjectId;
         {
             let mut reg = self.reg.borrow_mut();
             // An entry whose object died can't sit at this address: its
@@ -1001,44 +1002,105 @@ impl GcState {
             let Some(weak) = WeakObject::new(obj) else {
                 return false;
             };
-            // `finalized_ids` is keyed by object id (a pointer), which the
-            // allocator recycles. A freshly tracked object at a recycled
-            // address must start *un*-finalized (`test_is_finalized`).
-            // Almost always empty (only `__del__`-bearing objects ever land
-            // there).
-            let mut flags = 0;
-            // SAFETY: a read that runs no code.
-            if unsafe { self.finalized_ids.peek() }.is_none_or(|f| !f.is_empty()) {
-                let id = id_of(obj);
-                if fresh {
-                    self.finalized_ids.borrow_mut().remove(&id);
-                } else if self.finalized_ids.borrow().contains(&id) {
-                    flags = F_FINALIZED;
-                }
-            }
-            if !slot_is_intrusive(&weak) {
-                // Publish to the miss-filter *before* the insert becomes
-                // observable (we hold the registry borrow).
-                self.tracked_filter.insert(key);
+            // SAFETY: `obj` is alive.
+            unsafe { self.enter(&mut reg, weak, fresh) };
+        }
+        serial_add(&self.tracked_count, 1);
+        serial_add(&self.tracked_version, 1);
+        true
+    }
+
+    /// Enter the object `weak` names, which is alive and unregistered, in
+    /// generation 0.
+    ///
+    /// # Safety
+    ///
+    /// The object must be alive.
+    unsafe fn enter(&self, reg: &mut Registry, weak: WeakObject, fresh: bool) -> u32 {
+        let key = weak.addr() as ObjectId;
+        // `finalized_ids` is keyed by object id (a pointer), which the
+        // allocator recycles. A freshly tracked object at a recycled
+        // address must start *un*-finalized (`test_is_finalized`). Almost
+        // always empty (only `__del__`-bearing objects ever land there).
+        let mut flags = 0;
+        // SAFETY: a read that runs no code.
+        if unsafe { self.finalized_ids.peek() }.is_none_or(|f| !f.is_empty()) {
+            if fresh {
+                self.finalized_ids.borrow_mut().remove(&key);
+            } else if self.finalized_ids.borrow().contains(&key) {
+                flags = F_FINALIZED;
+            }
+        }
+        // (A pointer: the field lives in the object, not the handle, which
+        // moves into the registry below.)
+        // SAFETY: alive, per the caller.
+        let own = unsafe { weak_intrusive_slot(&weak) }.map(std::ptr::from_ref);
+        if own.is_none() {
+            // Publish to the miss-filter *before* the insert becomes
+            // observable (the caller holds the registry borrow).
+            self.tracked_filter.insert(key);
+        }
+        // (Registered other than through `track`, which retires the
+        // deferral itself.)
+        if let WeakObject::Instance(w) = &weak {
+            // SAFETY: alive, per the caller.
+            let i = unsafe { &*w.as_ptr() };
+            if i.is_gc_deferred() {
+                i.clear_deferred_tracking();
+            }
+        }
+        let slot = reg.insert(weak, 0, flags);
+        match own {
+            // SAFETY: the object is alive, per the caller.
+            Some(cell) => unsafe { (*cell).set(slot) },
+            None => {
+                reg.index.insert(key, slot);
             }
-            let slot = reg.insert(weak, 0, flags);
-            match intrusive_slot(obj) {
-                Some(cell) => cell.set(slot),
-                None => {
-                    reg.index.insert(key, slot);
+        }
+        slot
+    }
+
+    /// Register the young objects of `set` still alive (see
+    /// [`Self::young`]) under one registry borrow, keeping the set's
+    /// allocation for the next generation of young.
+    fn flush_set<T: 'static>(
+        &self,
+        set: &RefCell<Vec<crate::sync::Weak<T>>>,
+        wrap: fn(crate::sync::Weak<T>) -> WeakObject,
+    ) {
+        let Ok(mut young) = set.try_borrow_mut() else {
+            return;
+        };
+        if young.is_empty() {
+            return;
+        }
+        let mut added = 0;
+        {
+            let mut reg = self.reg.borrow_mut();
+            for w in young.drain(..) {
+                if w.strong_count() == 0 {
+                    continue;
                 }
-            }
-            // (Registered other than through `track`, which retires the
-            // deferral itself.)
-            if let Object::Instance(i) = obj {
-                if i.is_gc_deferred() {
-                    i.clear_deferred_tracking();
+                let weak = wrap(w);
+                // SAFETY (both): the object is alive (counted above, and
+                // nothing has run since).
+                let registered = match unsafe { weak_intrusive_slot(&weak) } {
+                    Some(own) => own.is_registered(),
+                    None => {
+                        let key = weak.addr() as ObjectId;
+                        self.tracked_filter.may_contain(key) && reg.index.contains_key(&key)
+                    }
+                };
+                if !registered {
+                    unsafe { self.enter(&mut reg, weak, false) };
+                    added += 1;
                 }
             }
         }
-        serial_add(&self.tracked_count, 1);
-        serial_add(&self.tracked_version, 1);
-        true
+        if added > 0 {
+            serial_add(&self.tracked_count, added);
+            serial_add(&self.tracked_version, 1);
+        }
     }
 
     /// Drop the registry entry of an instance that is dying (from its
@@ -1683,71 +1745,94 @@ impl GcState {
         // internal edges are accounted. A temporary is a candidate from its
         // first edge on, so each of its edges is tallied too. Temporaries
         // take part in the mark walk but are never reclaimed or tracked.
+        //
+        // The walk also records each candidate's edges to candidates (a
+        // node per registered candidate, in list order, then one per
+        // temporary), so marking what is reachable follows those instead of
+        // walking every object a second time.
         let mut found: Vec<Object> = Vec::new();
-        let tally = |obj: &Object, temps: &mut Vec<Temp>, found: &mut Vec<Object>| {
-            let parent_is_iter = matches!(obj, Object::Iter(_));
-            let parent_is_frame = matches!(obj, Object::Frame(_));
-            traverse_collected(obj, &mut |child| {
-                if child.never_gc_candidate() {
-                    return;
-                }
-                match finder.find(child) {
-                    Some(Hit::Real(s)) => {
-                        let e = reg.entry(s);
-                        e.gc_refs.set(e.gc_refs.get().wrapping_sub(1));
+        let mut edges: Vec<u32> = Vec::new();
+        let mut starts: Vec<u32> = Vec::new();
+        let tally =
+            |obj: &Object, temps: &mut Vec<Temp>, found: &mut Vec<Object>, edges: &mut Vec<u32>| {
+                let parent_is_iter = matches!(obj, Object::Iter(_));
+                let parent_is_frame = matches!(obj, Object::Frame(_));
+                traverse_collected(obj, &mut |child| {
+                    if child.never_gc_candidate() {
+                        return;
                     }
-                    Some(Hit::Temp(i)) => {
-                        let t = &temps[i as usize];
-                        t.gc_refs.set(t.gc_refs.get() - 1);
-                    }
-                    None => {
-                        if promotes_temporarily(child, parent_is_iter, parent_is_frame) {
-                            found.push(child.clone());
+                    match finder.find(child) {
+                        Some(Hit::Real(s)) => {
+                            let e = reg.entry(s);
+                            e.gc_refs.set(e.gc_refs.get().wrapping_sub(1));
+                            edges.push(s);
+                        }
+                        Some(Hit::Temp(i)) => {
+                            let t = &temps[i as usize];
+                            t.gc_refs.set(t.gc_refs.get() - 1);
+                            edges.push(i | TEMP_BIT);
+                        }
+                        None => {
+                            if promotes_temporarily(child, parent_is_iter, parent_is_frame) {
+                                found.push(child.clone());
+                            }
                         }
                     }
+                });
+                // (One entry per edge: a node reached twice is promoted once and
+                // tallied twice.)
+                for child in found.drain(..) {
+                    let i = match finder.find(&child) {
+                        Some(Hit::Temp(i)) => i as usize,
+                        Some(Hit::Real(s)) => {
+                            let e = reg.entry(s);
+                            e.gc_refs.set(e.gc_refs.get().wrapping_sub(1));
+                            edges.push(s);
+                            continue;
+                        }
+                        None => {
+                            finder.add_temp(&child, temps.len());
+                            temps.push(Temp {
+                                obj: child,
+                                gc_refs: std::cell::Cell::new(0),
+                                reached: std::cell::Cell::new(false),
+                            });
+                            temps.len() - 1
+                        }
+                    };
+                    let t = &temps[i];
+                    t.gc_refs.set(t.gc_refs.get() - 1);
+                    edges.push(i as u32 | TEMP_BIT);
                 }
-            });
-            // (One entry per edge: a node reached twice is promoted once and
-            // tallied twice.)
-            for child in found.drain(..) {
-                let i = match finder.find(&child) {
-                    Some(Hit::Temp(i)) => i as usize,
-                    Some(Hit::Real(s)) => {
-                        let e = reg.entry(s);
-                        e.gc_refs.set(e.gc_refs.get().wrapping_sub(1));
-                        continue;
-                    }
-                    None => {
-                        finder.add_temp(&child, temps.len());
-                        temps.push(Temp {
-                            obj: child,
-                            gc_refs: std::cell::Cell::new(0),
-                            reached: std::cell::Cell::new(false),
-                        });
-                        temps.len() - 1
-                    }
-                };
-                let t = &temps[i];
-                t.gc_refs.set(t.gc_refs.get() - 1);
-            }
-        };
+            };
         for &slot in lists.iter().flatten() {
+            let e = reg.entry(slot);
+            e.node.set(starts.len() as u32);
+            starts.push(edges.len() as u32);
             // SAFETY: the mark walk runs no code that could release the
             // object (see `traverse_collected`).
             unsafe {
-                reg.entry(slot)
-                    .obj
-                    .as_ref()
-                    .and_then(|w| w.with_borrowed(|obj| tally(obj, &mut temps, &mut found)));
+                e.obj.as_ref().and_then(|w| {
+                    w.with_borrowed(|obj| tally(obj, &mut temps, &mut found, &mut edges))
+                });
             }
         }
+        let n_real = starts.len();
         let mut scanned = 0;
         while scanned < temps.len() {
+            starts.push(edges.len() as u32);
             let obj = temps[scanned].obj.clone();
-            tally(&obj, &mut temps, &mut found);
+            tally(&obj, &mut temps, &mut found, &mut edges);
             scanned += 1;
         }
-        let lookup = |child: &Object| finder.find(child);
+        starts.push(edges.len() as u32);
+        let graph = Graph {
+            reg: &reg,
+            temps: &temps,
+            edges: &edges,
+            starts: &starts,
+            n_real,
+        };
 
         // Phase 3: add the outer refcount, read after the walk (an iterator
         // synthesises a fresh wrapper for its buffer on each traverse, alive
@@ -1809,7 +1894,7 @@ impl GcState {
                 kinds
             );
         }
-        Self::propagate_reached(&reg, &temps, &lookup, &mut grey);
+        graph.propagate(&mut grey);
 
         // Phase 5: the candidates still unreached are unreachable cyclic
         // garbage; hold each for the rest of the collection.
@@ -1913,7 +1998,7 @@ impl GcState {
                 protect.push(*slot);
             }
         }
-        Self::propagate_reached(&reg, &temps, &lookup, &mut protect);
+        graph.propagate(&mut protect);
 
         let dead: Vec<Object> = unreachable
             .iter()
@@ -1984,52 +2069,6 @@ impl GcState {
         reported
     }
 
-    /// Mark everything reachable from the candidates on the `grey` stack
-    /// (registry slots, or temporaries tagged [`TEMP_BIT`]), which are
-    /// marked already; `lookup` finds a child's candidate.
-    fn propagate_reached(
-        reg: &Registry,
-        temps: &[Temp],
-        lookup: &dyn Fn(&Object) -> Option<Hit>,
-        grey: &mut Vec<u32>,
-    ) {
-        while let Some(item) = grey.pop() {
-            let mut visit = |child: &Object| {
-                if child.never_gc_candidate() {
-                    return;
-                }
-                match lookup(child) {
-                    Some(Hit::Real(s)) => {
-                        let e = reg.entry(s);
-                        if !e.has(F_REACHED) {
-                            e.set(F_REACHED, true);
-                            grey.push(s);
-                        }
-                    }
-                    Some(Hit::Temp(i)) => {
-                        let t = &temps[i as usize];
-                        if !t.reached.get() {
-                            t.reached.set(true);
-                            grey.push(i | TEMP_BIT);
-                        }
-                    }
-                    None => {}
-                }
-            };
-            if item & TEMP_BIT == 0 {
-                // SAFETY: as in the mark walk.
-                unsafe {
-                    reg.entry(item)
-                        .obj
-                        .as_ref()
-                        .and_then(|w| w.with_borrowed(|obj| traverse_collected(obj, &mut visit)));
-                }
-            } else {
-                traverse_collected(&temps[(item & !TEMP_BIT) as usize].obj, &mut visit);
-            }
-        }
-    }
-
     /// Mark the live entries of generations `0..=upto` as candidates of a
     /// collection, dropping the entries whose object died. Returns how
     /// many there are.
@@ -2106,10 +2145,56 @@ impl GcState {
     }
 }
 
-/// How a collection's mark walk finds a child's candidate: the registry
-/// (which can't change while the walk runs), a cache in front of its id
-/// index for the children seen again and again (a module's namespace, a
-/// shared function), and the temporaries.
+/// A collection's candidates and their edges to one another, as the mark
+/// walk recorded them: node `k`'s edges are `edges[starts[k]..starts[k +
+/// 1]]`, each a registered candidate's slot or a temporary's position
+/// tagged [`TEMP_BIT`]; a registered candidate's node is in its entry,
+/// and temporary `i`'s is `n_real + i`.
+struct Graph<'a> {
+    reg: &'a Registry,
+    temps: &'a [Temp],
+    edges: &'a [u32],
+    starts: &'a [u32],
+    n_real: usize,
+}
+
+impl Graph<'_> {
+    /// Mark everything reachable from the candidates on the `grey` stack
+    /// (registry slots, or temporaries tagged [`TEMP_BIT`]), which are
+    /// marked already.
+    fn propagate(&self, grey: &mut Vec<u32>) {
+        while let Some(item) = grey.pop() {
+            let node = if item & TEMP_BIT == 0 {
+                self.reg.entry(item).node.get() as usize
+            } else {
+                self.n_real + (item & !TEMP_BIT) as usize
+            };
+            let range = self.starts[node] as usize..self.starts[node + 1] as usize;
+            for &edge in &self.edges[range] {
+                if edge & TEMP_BIT == 0 {
+                    let e = self.reg.entry(edge);
+                    if !e.has(F_REACHED) {
+                        e.set(F_REACHED, true);
+                        grey.push(edge);
+                    }
+                } else {
+                    let t = &self.temps[(edge & !TEMP_BIT) as usize];
+                    if !t.reached.get() {
+                        t.reached.set(true);
+                        grey.push(edge);
+                    }
+                }
+            }
+        }
+    }
+}
+
+/// How a collection's mark walk finds a child's candidate: through the
+/// child's own registry slot, or else a cache of answers by address in
+/// front of the registry's id index and the temporaries (a child is
+/// usually reached many times: a module's namespace from every frame, a
+/// class's method from every bound method). The registry can't change
+/// while the walk runs, and a new temporary updates its own cache line.
 struct Finder<'a> {
     reg: &'a Registry,
     filter: &'a crate::hot_filter::RebuildableBloom,
@@ -2118,13 +2203,13 @@ struct Finder<'a> {
     /// Some temporary is also a registered object (outside the
     /// candidates): a registered child may then still be a temporary.
     registered_temp: std::cell::Cell<bool>,
-    /// Payload address -> slot (or [`NO_SLOT`]), direct-mapped.
+    /// Payload address -> answer (see [`Finder::encode`]), direct-mapped.
     cache_keys: [std::cell::Cell<ObjectId>; FINDER_CACHE],
-    cache_slots: [std::cell::Cell<u32>; FINDER_CACHE],
+    cache_vals: [std::cell::Cell<u32>; FINDER_CACHE],
 }
 
 /// [`Finder`]'s cache size.
-const FINDER_CACHE: usize = 256;
+const FINDER_CACHE: usize = 512;
 
 impl<'a> Finder<'a> {
     fn new(reg: &'a Registry, filter: &'a crate::hot_filter::RebuildableBloom) -> Self {
@@ -2134,44 +2219,78 @@ impl<'a> Finder<'a> {
             temp_ids: std::cell::RefCell::new(GcIdMap::default()),
             registered_temp: std::cell::Cell::new(false),
             cache_keys: std::array::from_fn(|_| std::cell::Cell::new(0)),
-            cache_slots: std::array::from_fn(|_| std::cell::Cell::new(NO_SLOT)),
+            cache_vals: std::array::from_fn(|_| std::cell::Cell::new(NO_SLOT)),
         }
     }
 
-    /// `obj`'s registry slot, if it has one.
     #[inline]
-    fn slot(&self, obj: &Object) -> Option<u32> {
-        if let Some(cell) = intrusive_slot(obj) {
-            let s = cell.get();
-            return (s != NO_SLOT).then_some(s);
-        }
-        let key = obj.payload_addr()? as ObjectId;
-        if !self.filter.may_contain(key) {
-            return None;
+    fn line(key: ObjectId) -> usize {
+        ((key >> 4) ^ (key >> 13)) as usize & (FINDER_CACHE - 1)
+    }
+
+    /// An answer as a cache value: [`NO_SLOT`] for none, a registered
+    /// candidate's slot, or a temporary's position tagged [`TEMP_BIT`].
+    #[inline]
+    fn encode(hit: Option<Hit>) -> u32 {
+        match hit {
+            None => NO_SLOT,
+            Some(Hit::Real(s)) => s,
+            Some(Hit::Temp(i)) => i | TEMP_BIT,
         }
-        let i = ((key >> 4) ^ (key >> 12)) as usize & (FINDER_CACHE - 1);
-        let slot = if self.cache_keys[i].get() == key {
-            self.cache_slots[i].get()
+    }
+
+    #[inline]
+    fn decode(v: u32) -> Option<Hit> {
+        if v == NO_SLOT {
+            None
+        } else if v & TEMP_BIT != 0 {
+            Some(Hit::Temp(v & !TEMP_BIT))
         } else {
-            let slot = self.reg.index.get(&key).copied().unwrap_or(NO_SLOT);
-            self.cache_keys[i].set(key);
-            self.cache_slots[i].set(slot);
-            slot
-        };
-        (slot != NO_SLOT).then_some(slot)
+            Some(Hit::Real(v))
+        }
     }
 
     /// `child`'s candidate, if it is one.
     #[inline]
     fn find(&self, child: &Object) -> Option<Hit> {
-        if let Some(slot) = self.slot(child) {
-            if self.reg.entry(slot).has(F_CAND) {
-                return Some(Hit::Real(slot));
+        if let Some(cell) = intrusive_slot(child) {
+            let s = cell.get();
+            if s != NO_SLOT {
+                if self.reg.entry(s).has(F_CAND) {
+                    return Some(Hit::Real(s));
+                }
+                if !self.registered_temp.get() {
+                    return None;
+                }
             }
-            if !self.registered_temp.get() {
+            // (Of these kinds, only an instance can be a temporary.)
+            if !matches!(child, Object::Instance(_)) {
                 return None;
             }
         }
+        let key = child.payload_addr()? as ObjectId;
+        let line = Self::line(key);
+        if self.cache_keys[line].get() == key {
+            return Self::decode(self.cache_vals[line].get());
+        }
+        let hit = self.find_uncached(child, key);
+        self.cache_keys[line].set(key);
+        self.cache_vals[line].set(Self::encode(hit));
+        hit
+    }
+
+    /// [`Self::find`] past the intrusive slot and the cache.
+    fn find_uncached(&self, child: &Object, key: ObjectId) -> Option<Hit> {
+        if intrusive_slot(child).is_none() && self.filter.may_contain(key) {
+            if let Some(&s) = self.reg.index.get(&key) {
+                if self.reg.entry(s).has(F_CAND) {
+                    return Some(Hit::Real(s));
+                }
+                if !self.registered_temp.get() {
+                    return None;
+                }
+            }
+        }
         if !may_be_temporary(child) {
             return None;
         }
@@ -2182,12 +2301,28 @@ impl<'a> Finder<'a> {
         ids.get(&id_of(child)).map(|&i| Hit::Temp(i as u32))
     }
 
+    /// Whether `obj` has a registry entry.
+    fn registered(&self, obj: &Object) -> bool {
+        self.reg.slot_of(obj, self.filter).is_some()
+    }
+
     /// Record `child` as the temporary at position `i`.
     fn add_temp(&self, child: &Object, i: usize) {
-        if self.slot(child).is_some() {
+        if !self.registered_temp.get() && self.registered(child) {
+            // The cached "not a candidate" answers for registered
+            // objects no longer hold.
             self.registered_temp.set(true);
+            for k in &self.cache_keys {
+                k.set(0);
+            }
         }
         self.temp_ids.borrow_mut().insert(id_of(child), i);
+        if let Some(key) = child.payload_addr() {
+            let key = key as ObjectId;
+            let line = Self::line(key);
+            self.cache_keys[line].set(key);
+            self.cache_vals[line].set(Self::encode(Some(Hit::Temp(i as u32))));
+        }
     }
 }
 
diff --git a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
index b54f5b90..6695b0fc 100644
--- a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
@@ -642,6 +642,19 @@ fn fut_gc_matches(obj: &Object) -> bool {
     }
 }
 
+/// [`fut_gc_matches`]'s cheap superset for the traverse hook, which every
+/// instance a collection walks is offered: an instance with slot storage
+/// (a future's handle lives in a slot). [`fut_gc_traverse`] finds the
+/// state itself, so the precise lookup runs once.
+fn fut_gc_may_match(obj: &Object) -> bool {
+    match obj {
+        // SAFETY: a read that runs no code. (A borrowed cell answers
+        // "maybe".)
+        Object::Instance(i) => unsafe { i.slots.peek() }.is_none_or(|s| !s.is_empty_default()),
+        _ => false,
+    }
+}
+
 fn fut_gc_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) {
     let Object::Instance(inst) = obj else { return };
     let Some(st) = existing_state_of(inst) else {
@@ -730,7 +743,7 @@ fn register_gc_hooks() {
     static ONCE: std::sync::Once = std::sync::Once::new();
     ONCE.call_once(|| {
         let kind = crate::gc_trace::hook_kind::INSTANCE;
-        crate::gc_trace::register_traverse_for(kind, fut_gc_matches, fut_gc_traverse);
+        crate::gc_trace::register_traverse_for(kind, fut_gc_may_match, fut_gc_traverse);
         crate::gc_trace::register_clear_for(kind, fut_gc_matches, fut_gc_clear);
     });
 }

From e62332cf8bc163329dbafdb53d0d88aade5cc857 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 02:15:15 -0700
Subject: [PATCH 031/173] perf: cheaper container deferral and sweeps

A fresh list, dict or set holding only atomic values is remembered
weakly instead of tracked. Its scan and the deferral list now take
guard-free views of their cells (neither runs code), and a sweep of
the deferrals no longer flushes the young instance, function and
generator sets: a sweep that compacts deferrals as they pile up
registered every young object with them, which a program building a
long-lived structure (an element tree) paid for every 4096 containers.
Only a collection or a reflective API flushes the young sets now. A
sweep also hands its promotions to the registry under one borrow,
moving each weak handle into its entry instead of upgrading and
re-registering the container.

Suite vs the previous commit (ins_suite, pinned seed): xml_etree
0.935, comprehensions 0.973, pickle_roundtrip 0.989, geomean 0.9960
(against m1: geomean 0.9906).
---
 crates/weavepy-vm/src/gc_trace.rs | 169 ++++++++++++++++++++++--------
 1 file changed, 127 insertions(+), 42 deletions(-)

diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs
index e33f55b6..ec58fd9e 100644
--- a/crates/weavepy-vm/src/gc_trace.rs
+++ b/crates/weavepy-vm/src/gc_trace.rs
@@ -887,11 +887,18 @@ impl GcState {
             return false;
         };
         let over = {
-            let Ok(mut deferred) = self.deferred.try_borrow_mut() else {
-                // A sweep is already walking the list (it upgrades and can
-                // re-enter through a promotion). Track eagerly rather than
-                // queue onto a list we cannot touch.
-                return false;
+            // SAFETY: nothing below runs code while the list is borrowed
+            // (a dead predecessor's release only frees memory).
+            let mut guard = None;
+            let deferred = match unsafe { self.deferred.peek_mut() } {
+                Some(d) => d,
+                None => match self.deferred.try_borrow_mut() {
+                    Ok(g) => &mut **guard.insert(g),
+                    // A sweep is already walking the list (it upgrades and
+                    // can re-enter through a promotion). Track eagerly
+                    // rather than queue onto a list we cannot touch.
+                    Err(_) => return false,
+                },
             };
             // The churn shape — a loop that builds a container and drops
             // the previous one — leaves its dead predecessors just below
@@ -930,9 +937,12 @@ impl GcState {
     /// [`Self::sweep_deferred`] of the young deferrals, and of the old ones
     /// too when `old` (see [`Self::deferred_old`]); the young survivors
     /// join the old.
+    ///
+    /// The young sets are left alone: a sweep that compacts the deferrals
+    /// as they accumulate must not register every young object with them
+    /// (a collection, or a reflective API, flushes them itself).
     fn sweep_deferred_from(&self, promote_all: bool, old: bool) {
-        self.flush_young();
-        let mut promote: Vec<Object> = Vec::new();
+        let mut promote: Vec<WeakObject> = Vec::new();
         {
             let (Ok(mut deferred), Ok(mut aged)) = (
                 self.deferred.try_borrow_mut(),
@@ -940,21 +950,28 @@ impl GcState {
             ) else {
                 return;
             };
-            let mut keep = |entry: &DeferredContainer| {
-                let Some(obj) = entry.upgrade() else {
-                    return false;
-                };
-                if promote_all || container_can_cycle(&obj) {
-                    promote.push(obj);
-                    return false;
+            // Keep the live entries of `from` that still hold only atomic
+            // values (appending them to `keep`); promote the rest.
+            let mut sift = |from: Vec<DeferredContainer>, keep: &mut Vec<DeferredContainer>| {
+                for entry in from {
+                    if entry.is_dead() {
+                        continue;
+                    }
+                    // SAFETY: alive (checked above), and the scan runs no
+                    // code.
+                    if promote_all || unsafe { entry.with_borrowed(container_can_cycle) } {
+                        promote.push(entry.into_weak());
+                    } else {
+                        keep.push(entry);
+                    }
                 }
-                true
             };
-            deferred.retain(&mut keep);
             if old {
-                aged.retain(&mut keep);
+                let from = std::mem::take(&mut *aged);
+                sift(from, &mut aged);
             }
-            aged.append(&mut deferred);
+            let from = std::mem::take(&mut *deferred);
+            sift(from, &mut aged);
             if aged.len() >= DEFERRED_CAP && !old {
                 // (Dead entries first: the old ones were left unswept.)
                 aged.retain(|e| !e.is_dead());
@@ -962,18 +979,50 @@ impl GcState {
             if aged.len() >= DEFERRED_CAP {
                 // The set has stopped being a churn buffer: hand it all
                 // over, so these allocations resume pacing collections.
-                promote.extend(aged.drain(..).filter_map(|e| e.upgrade()));
+                promote.extend(
+                    aged.drain(..)
+                        .filter(|e| !e.is_dead())
+                        .map(DeferredContainer::into_weak),
+                );
             }
             self.deferred_limit.store(DEFERRED_FLOOR, Ordering::Relaxed);
         }
-        for obj in promote {
-            self.track_now(&obj);
+        if !promote.is_empty() {
+            self.register_weaks(promote);
+        }
+    }
+
+    /// [`Self::track_now`] for the live objects `weaks` names (promoted
+    /// deferrals), under one registry borrow.
+    fn register_weaks(&self, weaks: Vec<WeakObject>) {
+        let mut added = 0;
+        {
+            let mut reg = self.reg.borrow_mut();
+            for weak in weaks {
+                if weak.is_dead() {
+                    continue;
+                }
+                // (None of these kinds records its own slot.)
+                let key = weak.addr() as ObjectId;
+                if self.tracked_filter.may_contain(key) && reg.index.contains_key(&key) {
+                    continue;
+                }
+                // SAFETY: alive (checked above, and nothing has run since).
+                unsafe { self.enter(&mut reg, weak, true) };
+                added += 1;
+            }
+        }
+        if added > 0 {
+            serial_add(&self.tracked_count, added);
+            serial_add(&self.tracked_version, 1);
+            self.note_gen0_allocs(added);
         }
     }
 
     /// Hand every deferred container to the collector. For the reflective
     /// APIs, which must enumerate the same population CPython does.
     pub fn promote_all_deferred(&self) {
+        self.flush_young();
         self.sweep_deferred(true);
     }
 
@@ -1414,6 +1463,23 @@ impl GcState {
         }
     }
 
+    /// [`Self::note_gen0_alloc`] of `n` allocations at once.
+    fn note_gen0_allocs(&self, n: usize) {
+        let before = {
+            let mut counts = self.counts.borrow_mut();
+            let before = counts[0];
+            counts[0] = counts[0].saturating_add(n);
+            self.sync_gen0_gauge(counts[0], None);
+            before
+        };
+        let gauge = self.gen0_gauge.load(Ordering::Relaxed);
+        let threshold = (gauge & 0xffff_ffff) as usize;
+        let now = (gauge >> 32) as usize;
+        if threshold != 0 && before < threshold && now >= threshold && self.is_enabled() {
+            crate::hot_gates::set(crate::hot_gates::GC_DUE);
+        }
+    }
+
     /// Republish the gen-0 gauge after a write to `counts[0]` or
     /// `thresholds[0]` (see the field docs).
     #[inline]
@@ -1600,6 +1666,7 @@ impl GcState {
     /// *without* the destructive teardown of a real collection. See
     /// [`Self::collect_generation`]'s `weakref_only` discussion.
     pub fn fire_dead_weakrefs(&self) {
+        self.flush_young();
         self.sweep_deferred_from(false, true);
         if self
             .collecting
@@ -1628,6 +1695,7 @@ impl GcState {
         // the mark phase sees the whole candidate population (the old
         // deferrals only for the older generations). Before the
         // re-entrancy claim: promotion calls `track_now`.
+        self.flush_young();
         self.sweep_deferred_from(false, upto >= 1);
         // Atomic claim: overlapping collections over the shared heap would
         // subtract the same internal edges twice.
@@ -3475,14 +3543,32 @@ impl DeferredContainer {
         }
     }
 
-    /// The container, if it is still alive.
-    fn upgrade(&self) -> Option<Object> {
+    /// The handle as the registry holds it.
+    fn into_weak(self) -> WeakObject {
         match self {
-            Self::List(w) => w.upgrade().map(Object::List),
-            Self::Dict(w) => w.upgrade().map(Object::Dict),
-            Self::Set(w) => w.upgrade().map(Object::Set),
+            Self::List(w) => WeakObject::List(w),
+            Self::Dict(w) => WeakObject::Dict(w),
+            Self::Set(w) => WeakObject::Set(w),
         }
     }
+
+    /// Run `f` on the container through a handle that takes no reference.
+    ///
+    /// # Safety
+    ///
+    /// The container must be alive, and nothing `f` does may release it.
+    unsafe fn with_borrowed<R>(&self, f: impl FnOnce(&Object) -> R) -> R {
+        // SAFETY: alive, per the caller; the view owns no reference and is
+        // never dropped.
+        let obj = std::mem::ManuallyDrop::new(unsafe {
+            match self {
+                Self::List(w) => Object::List(crate::Rc::from_raw(w.as_ptr())),
+                Self::Dict(w) => Object::Dict(crate::Rc::from_raw(w.as_ptr())),
+                Self::Set(w) => Object::Set(crate::Rc::from_raw(w.as_ptr())),
+            }
+        });
+        f(&obj)
+    }
 }
 
 /// An element that cannot route a cycle back out of the container
@@ -3505,23 +3591,22 @@ fn container_can_cycle(obj: &Object) -> bool {
     // *container*, so beyond a point it stops paying for itself — and an
     // unbounded scan would make `track` O(len) for `list(range(1e6))`.
     const SCAN_CAP: usize = 32;
+    // (The scans run no code: a guard-free read is enough.)
     match obj {
-        Object::List(l) => l
-            .try_borrow()
-            .map(|v| v.len() > SCAN_CAP || v.iter().any(|x| !element_is_inert(x)))
-            .unwrap_or(true),
-        Object::Set(s) => s
-            .try_borrow()
-            .map(|m| m.len() > SCAN_CAP || m.iter().any(|k| !element_is_inert(&k.0)))
-            .unwrap_or(true),
-        Object::Dict(d) => d
-            .try_borrow()
-            .map(|m| {
-                m.len() > SCAN_CAP
-                    || m.iter()
-                        .any(|(k, v)| !element_is_inert(&k.0) || !element_is_inert(v))
-            })
-            .unwrap_or(true),
+        Object::List(l) => with_read::<true, _, _>(l, |v| {
+            v.len() > SCAN_CAP || v.iter().any(|x| !element_is_inert(x))
+        })
+        .unwrap_or(true),
+        Object::Set(s) => with_read::<true, _, _>(s, |m| {
+            m.len() > SCAN_CAP || m.iter().any(|k| !element_is_inert(&k.0))
+        })
+        .unwrap_or(true),
+        Object::Dict(d) => with_read::<true, _, _>(d, |m| {
+            m.len() > SCAN_CAP
+                || m.iter()
+                    .any(|(k, v)| !element_is_inert(&k.0) || !element_is_inert(v))
+        })
+        .unwrap_or(true),
         // A tuple can only anchor a cycle through a non-atomic element. An
         // empty or all-scalar tuple (the interned `()`, `(1, 2)`, …) can never
         // close one, so it stays off the GC's books.

From 5051ac481655dee569980e994162cf7fa8b091c6 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 00:43:14 -0700
Subject: [PATCH 032/173] perf: replace IndexMap with a compact dict table

DictData now wraps dictmap::DictMap, a purpose-built ordered table in
the shape of CPython's compact dict: a dense entry vector carrying each
key's Python hash, plus a slot table of one word per slot (entry index
and half of the mixed hash) probed linearly, with backward-shift removal
and no tombstones. Tables of up to eight entries have no slots and scan
their entries. A probe compares stored hashes before keys, so a key's
__eq__ only ever meets keys of equal hash, as in CPython.

Probes go through a Probe trait in place of Hash + Equivalent. The str
and int arms of LeafProbe are inlined; a probe with a str whose hash is
not yet known compares a small table's keys directly instead of hashing
(StrKey, fresh-string DictKeys). Key strings compare in line instead of
via memcmp. Stores that miss now insert through the probe's vacancy (one
probe, not two). The raw-entry users (reentrant probes, split-dict
materialization, pickle, tier 2 stores) use find_by_hash and
insert_unique_hashed on the stored Python hash.

Positions stay dense, so get_index, the position caches and the
iterators are unchanged.

Instructions per call, pinned seed, against the previous commit: suite
geomean 0.983 (1.586x -> 1.560x CPython); collections_mix 0.938,
scimark 0.938, json_roundtrip 0.951, pickle_roundtrip 0.953, sudoku
0.956, deepcopy 0.973, closures_decorators 0.973, graph_search 0.98,
logging_mix 0.981.
Peak RSS within +-3% on every workload.
---
 crates/weavepy-vm/src/builtins.rs             |   34 +-
 crates/weavepy-vm/src/dictmap.rs              | 1278 +++++++++++++++++
 crates/weavepy-vm/src/frame_jit.rs            |   10 +-
 crates/weavepy-vm/src/inst_dict.rs            |   10 +-
 crates/weavepy-vm/src/lib.rs                  |   54 +-
 crates/weavepy-vm/src/object.rs               |  241 ++--
 crates/weavepy-vm/src/shared_value.rs         |    6 +
 crates/weavepy-vm/src/specialize.rs           |    2 +-
 .../src/stdlib/collections_native.rs          |   23 +-
 .../src/stdlib/collections_odict.rs           |    2 +-
 .../src/stdlib/elementtree_native.rs          |   10 +-
 crates/weavepy-vm/src/stdlib/pickle_accel.rs  |   16 +-
 .../src/stdlib/pickle_accel/classes.rs        |    8 +-
 crates/weavepy-vm/src/tier2.rs                |   16 +-
 tests/regrtest/test_dict_table.py             |  187 +++
 15 files changed, 1693 insertions(+), 204 deletions(-)
 create mode 100644 crates/weavepy-vm/src/dictmap.rs
 create mode 100644 tests/regrtest/test_dict_table.py

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index e7470fc5..8526cd18 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -13506,21 +13506,25 @@ pub(crate) fn dict_insert_exact(
     let Ok(mut m) = d.try_borrow_mut() else {
         return Err((key, value));
     };
-    if let Some(slot) = m.get_mut(&probe) {
-        let replaced = !slot.is_same(&value);
-        let old = std::mem::replace(slot, value);
-        drop(m);
-        // (Re-storing the identical object is no change; PEP 509.)
-        if replaced {
-            crate::object::dict_mutation_event(d);
+    let vacancy = match m.probe_entry(&probe) {
+        crate::dictmap::ProbeEntry::Occupied(e) => {
+            let slot = e.into_mut();
+            let replaced = !slot.is_same(&value);
+            let old = std::mem::replace(slot, value);
+            drop(m);
+            // (Re-storing the identical object is no change; PEP 509.)
+            if replaced {
+                crate::object::dict_mutation_event(d);
+            }
+            return Ok(Some(old));
         }
-        return Ok(Some(old));
-    }
+        crate::dictmap::ProbeEntry::Vacant(e) => e,
+    };
     if !probe.miss_is_exact() {
         drop(m);
         return Err((key, value));
     }
-    m.insert(DictKey(key), value);
+    vacancy.insert(DictKey(key), value);
     drop(m);
     crate::object::dict_watch_bump(d);
     crate::object::dict_mutation_event(d);
@@ -13810,17 +13814,17 @@ pub(crate) fn dict_setitem_leaf(args: &[Object]) -> Option<Result<Object, Runtim
     let probe = crate::object::LeafProbe::new(key)?;
     let cell = leaf_dict_of(recv)?;
     let mut d = cell.try_borrow_mut().ok()?;
-    let (old, changed, added) = match d.get_mut(&probe) {
-        Some(slot) => {
-            let old = std::mem::replace(slot, value.clone());
+    let (old, changed, added) = match d.probe_entry(&probe) {
+        crate::dictmap::ProbeEntry::Occupied(e) => {
+            let old = std::mem::replace(e.into_mut(), value.clone());
             let changed = !old.is_same(value);
             (Some(old), changed, false)
         }
-        None => {
+        crate::dictmap::ProbeEntry::Vacant(e) => {
             if !probe.miss_is_exact() {
                 return None;
             }
-            d.insert(DictKey(key.clone()), value.clone());
+            e.insert(DictKey(key.clone()), value.clone());
             (None, true, true)
         }
     };
diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
new file mode 100644
index 00000000..ae6ddf71
--- /dev/null
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -0,0 +1,1278 @@
+//! The insertion-ordered hash table behind every Python `dict`.
+//!
+//! The layout follows CPython's compact dict: a dense vector of entries in
+//! insertion order, each carrying its key's Python hash, plus a sparse
+//! table of slots that point into it. A table of at most [`SMALL`] entries
+//! has no slots at all: a probe scans the entries, comparing hashes first,
+//! so the commonest dicts (keyword arguments, small literals, instance
+//! dictionaries) are one allocation.
+//!
+//! A slot is one word: the entry's index plus one in the low half (zero
+//! marks an empty slot) and the high half of the mixed hash (the key's
+//! Python hash times a Fibonacci constant) above it. The top bits of the
+//! mixed hash pick the home slot, and collisions probe linearly; a probe
+//! compares the slot's half-hash before it touches an entry, and an entry
+//! only when the full stored hash matches, so a key's `__eq__` runs only
+//! against keys of equal hash, as in CPython. Removal shifts the run that
+//! follows a freed slot back over it, so there are no tombstones.
+//!
+//! Positions are dense: removing an entry shifts the later ones down, so
+//! `get_index(i)` is always the `i`-th key in order (the contract the
+//! iterators and the position caches rely on).
+//!
+//! A lookup takes any [`Probe`]: a stored key's own type, or one of the
+//! borrowed probes in [`crate::object`] (a `&str` name, a native-equality
+//! leaf probe) that hash and compare without building an `Object`.
+
+use crate::object::{DictKey, Object};
+use std::fmt;
+
+/// The most entries a table keeps without slots.
+pub const SMALL: usize = 8;
+
+/// The Fibonacci multiplier that mixes a Python hash into slot bits.
+const MIX: u64 = 0x9E37_79B9_7F4A_7C15;
+
+/// The low half of a slot: the entry index plus one.
+const IX_MASK: u64 = 0xFFFF_FFFF;
+
+/// A stored entry: the key's Python hash, the key, and its value.
+#[derive(Clone, Debug)]
+pub struct Bucket {
+    pub(crate) hash: i64,
+    pub key: DictKey,
+    pub value: Object,
+}
+
+/// A key a table can be probed with: its Python hash and its equality with
+/// a stored key of the same hash.
+pub trait Probe {
+    /// The probe's Python hash.
+    fn probe_hash(&self) -> i64;
+    /// Whether the probe equals `stored` (whose hash equals the probe's).
+    fn probe_eq(&self, stored: &DictKey) -> bool;
+
+    /// Whether the probe equals `stored` whatever its hash, when that is
+    /// settled natively and costs less than hashing the probe; `None`
+    /// when the hashes must be compared first. A small table is scanned
+    /// with this before the probe is hashed (a `str` probe whose hash
+    /// isn't known compares bytes instead of hashing them).
+    #[inline(always)]
+    fn probe_eq_unhashed(&self, _stored: &DictKey) -> Option<bool> {
+        None
+    }
+}
+
+impl Probe for DictKey {
+    #[inline]
+    fn probe_hash(&self) -> i64 {
+        crate::object::dict_key_hash(&self.0)
+    }
+
+    #[inline]
+    fn probe_eq(&self, stored: &DictKey) -> bool {
+        self == stored
+    }
+
+    #[inline]
+    fn probe_eq_unhashed(&self, stored: &DictKey) -> Option<bool> {
+        // A new string (its hash not yet computed) compares with the
+        // stored strings instead.
+        match (&self.0, &stored.0) {
+            (Object::Str(a), Object::Str(b)) if crate::shared_value::SharedStr::known_hash(a).is_none() => {
+                Some(a.as_bytes() == b.as_bytes())
+            }
+            (Object::Str(a), Object::WStr(_)) if crate::shared_value::SharedStr::known_hash(a).is_none() => {
+                Some(false)
+            }
+            _ => None,
+        }
+    }
+}
+
+impl<P: Probe + ?Sized> Probe for &P {
+    #[inline]
+    fn probe_hash(&self) -> i64 {
+        (**self).probe_hash()
+    }
+
+    #[inline]
+    fn probe_eq(&self, stored: &DictKey) -> bool {
+        (**self).probe_eq(stored)
+    }
+
+    #[inline]
+    fn probe_eq_unhashed(&self, stored: &DictKey) -> Option<bool> {
+        (**self).probe_eq_unhashed(stored)
+    }
+}
+
+/// Byte equality for key strings, in line: a probe that meets an equal
+/// stored string of another object compares a few words instead of
+/// calling `memcmp` (keys are mostly short names).
+#[inline(always)]
+pub fn key_bytes_eq(a: &[u8], b: &[u8]) -> bool {
+    let n = a.len();
+    if n != b.len() {
+        return false;
+    }
+    // SAFETY: every read is inside both slices (`n` bytes each): the two
+    // loads of a width overlap when `n` is below twice it.
+    unsafe {
+        let (p, q) = (a.as_ptr(), b.as_ptr());
+        if n >= 8 {
+            if n > 16 {
+                return a == b;
+            }
+            let x = p.cast::<u64>().read_unaligned() ^ q.cast::<u64>().read_unaligned();
+            let y = p.add(n - 8).cast::<u64>().read_unaligned()
+                ^ q.add(n - 8).cast::<u64>().read_unaligned();
+            return (x | y) == 0;
+        }
+        if n >= 4 {
+            let x = p.cast::<u32>().read_unaligned() ^ q.cast::<u32>().read_unaligned();
+            let y = p.add(n - 4).cast::<u32>().read_unaligned()
+                ^ q.add(n - 4).cast::<u32>().read_unaligned();
+            return (x | y) == 0;
+        }
+        if n >= 2 {
+            let x = p.cast::<u16>().read_unaligned() ^ q.cast::<u16>().read_unaligned();
+            let y = p.add(n - 2).cast::<u16>().read_unaligned()
+                ^ q.add(n - 2).cast::<u16>().read_unaligned();
+            return (x | y) == 0;
+        }
+        n == 0 || *p == *q
+    }
+}
+
+/// An ordered hash table from [`DictKey`] to [`Object`] (see the module
+/// docs).
+pub struct DictMap {
+    entries: Vec<Bucket>,
+    /// The slot table: empty for a small table, else a power of two long.
+    slots: Box<[u64]>,
+    /// `64 - log2(slots.len())`: the shift that takes a mixed hash to its
+    /// home slot.
+    shift: u32,
+}
+
+#[inline(always)]
+fn mix(hash: i64) -> u64 {
+    (hash as u64).wrapping_mul(MIX)
+}
+
+/// The slot count for `n` entries: at most two thirds full.
+fn slots_for(n: usize) -> usize {
+    (n.max(SMALL + 1) * 3 / 2 + 1).next_power_of_two()
+}
+
+impl Default for DictMap {
+    #[inline]
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Clone for DictMap {
+    fn clone(&self) -> Self {
+        Self {
+            entries: self.entries.clone(),
+            slots: self.slots.clone(),
+            shift: self.shift,
+        }
+    }
+}
+
+impl fmt::Debug for DictMap {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_map()
+            .entries(self.entries.iter().map(|b| (&b.key, &b.value)))
+            .finish()
+    }
+}
+
+impl DictMap {
+    /// An empty table.
+    #[inline]
+    pub fn new() -> Self {
+        Self {
+            entries: Vec::new(),
+            slots: Box::new([]),
+            shift: 64,
+        }
+    }
+
+    /// An empty table with room for `n` entries.
+    pub fn with_capacity(n: usize) -> Self {
+        let mut m = Self {
+            entries: Vec::with_capacity(n),
+            slots: Box::new([]),
+            shift: 64,
+        };
+        if n > SMALL {
+            m.rebuild_slots(slots_for(n));
+        }
+        m
+    }
+
+    /// [`Self::with_capacity`]; the hasher is implied (the table mixes
+    /// Python hashes itself).
+    #[inline]
+    pub fn with_capacity_and_hasher(n: usize, _: crate::fasthash::FxBuildHasher) -> Self {
+        Self::with_capacity(n)
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.entries.len()
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.entries.is_empty()
+    }
+
+    /// How many entries fit before the entry vector reallocates.
+    #[inline]
+    pub fn capacity(&self) -> usize {
+        self.entries.capacity()
+    }
+
+    /// The entries in order.
+    #[inline]
+    pub fn as_entries(&self) -> &[Bucket] {
+        &self.entries
+    }
+
+    /// Make room for `additional` more entries.
+    pub fn reserve(&mut self, additional: usize) {
+        self.entries.reserve(additional);
+        self.reserve_slots(additional);
+    }
+
+    /// [`Self::reserve`], reporting an allocation failure instead of
+    /// aborting.
+    pub fn try_reserve(&mut self, additional: usize) -> Result<(), std::collections::TryReserveError> {
+        self.entries.try_reserve(additional)?;
+        self.reserve_slots(additional);
+        Ok(())
+    }
+
+    fn reserve_slots(&mut self, additional: usize) {
+        let want = self.entries.len().saturating_add(additional);
+        if want > SMALL && (want * 3 > self.slots.len() * 2) {
+            self.rebuild_slots(slots_for(want));
+        }
+    }
+
+    /// Release spare capacity.
+    pub fn shrink_to_fit(&mut self) {
+        self.entries.shrink_to_fit();
+        if self.entries.len() <= SMALL {
+            self.slots = Box::new([]);
+            self.shift = 64;
+        } else if slots_for(self.entries.len()) < self.slots.len() {
+            self.rebuild_slots(slots_for(self.entries.len()));
+        }
+    }
+
+    /// Remove every entry, keeping the allocations.
+    pub fn clear(&mut self) {
+        self.entries.clear();
+        self.slots.fill(0);
+    }
+
+    /// Rebuild the slot table at `n` slots (a power of two) from the
+    /// entries' stored hashes; no key is hashed or compared.
+    fn rebuild_slots(&mut self, n: usize) {
+        debug_assert!(n.is_power_of_two() && n >= 16);
+        assert!(
+            n <= 1 << 32 && self.entries.len() < IX_MASK as usize,
+            "dict too large"
+        );
+        self.slots = vec![0u64; n].into_boxed_slice();
+        self.shift = 64 - n.trailing_zeros();
+        let mask = n - 1;
+        for (i, b) in self.entries.iter().enumerate() {
+            let m = mix(b.hash);
+            let mut pos = (m >> self.shift) as usize;
+            // SAFETY: `pos` is masked into the table throughout.
+            unsafe {
+                while *self.slots.get_unchecked(pos) != 0 {
+                    pos = (pos + 1) & mask;
+                }
+                *self.slots.get_unchecked_mut(pos) = (m & !IX_MASK) | (i as u64 + 1);
+            }
+        }
+    }
+
+    /// Re-derive the slots after the entries were reordered or dropped in
+    /// bulk.
+    fn reindex(&mut self) {
+        if self.slots.is_empty() {
+            return;
+        }
+        self.rebuild_slots(self.slots.len());
+    }
+
+    /// The position of the entry equal to `key` with hash `hash`, or, on a
+    /// miss, `Err` with the empty slot where it would go (unused for a
+    /// small table).
+    #[inline(always)]
+    fn find_hashed<Q: Probe + ?Sized>(&self, hash: i64, key: &Q) -> Result<usize, usize> {
+        if self.slots.is_empty() {
+            for (i, b) in self.entries.iter().enumerate() {
+                if b.hash == hash && key.probe_eq(&b.key) {
+                    return Ok(i);
+                }
+            }
+            return Err(0);
+        }
+        let m = mix(hash);
+        let mask = self.slots.len() - 1;
+        let mut pos = (m >> self.shift) as usize;
+        loop {
+            // SAFETY: `pos` is masked into the table, and a nonzero slot
+            // holds a live entry index plus one.
+            unsafe {
+                let s = *self.slots.get_unchecked(pos);
+                if s == 0 {
+                    return Err(pos);
+                }
+                if (s ^ m) <= IX_MASK {
+                    let ix = (s & IX_MASK) as usize - 1;
+                    let b = self.entries.get_unchecked(ix);
+                    if b.hash == hash && key.probe_eq(&b.key) {
+                        return Ok(ix);
+                    }
+                }
+            }
+            pos = (pos + 1) & mask;
+        }
+    }
+
+    /// The slot that holds entry `ix` (whose hash is `hash`).
+    #[inline]
+    fn slot_of(&self, hash: i64, ix: usize) -> usize {
+        let m = mix(hash);
+        let mask = self.slots.len() - 1;
+        let mut pos = (m >> self.shift) as usize;
+        let want = ix as u64 + 1;
+        loop {
+            // SAFETY: masked into the table; entry `ix` is present, so
+            // the walk ends at its slot before any empty one.
+            let s = unsafe { *self.slots.get_unchecked(pos) };
+            debug_assert!(s != 0, "entry {ix} has no slot");
+            if s & IX_MASK == want {
+                return pos;
+            }
+            pos = (pos + 1) & mask;
+        }
+    }
+
+    /// Free slot `pos`, shifting back the run after it (no tombstones).
+    fn erase_slot(&mut self, mut pos: usize) {
+        let mask = self.slots.len() - 1;
+        let bits = 64 - self.shift;
+        let mut next = (pos + 1) & mask;
+        loop {
+            let s = self.slots[next];
+            if s == 0 {
+                break;
+            }
+            let home = (s >> (64 - bits)) as usize;
+            // The entry at `next` may fill the hole when its home is not
+            // strictly between the hole and itself.
+            if (next.wrapping_sub(home) & mask) >= (next.wrapping_sub(pos) & mask) {
+                self.slots[pos] = s;
+                pos = next;
+            }
+            next = (next + 1) & mask;
+        }
+        self.slots[pos] = 0;
+    }
+
+    /// Point the slot of entry `from` (hash `hash`) at entry `to`.
+    #[inline]
+    fn repoint(&mut self, hash: i64, from: usize, to: usize) {
+        let pos = self.slot_of(hash, from);
+        self.slots[pos] = (self.slots[pos] & !IX_MASK) | (to as u64 + 1);
+    }
+
+    // ----------------------------------------------------------------
+    // Lookups.
+
+    /// The position of the entry equal to `key`.
+    #[inline]
+    pub fn get_index_of<Q: Probe + ?Sized>(&self, key: &Q) -> Option<usize> {
+        if self.slots.is_empty() {
+            if self.entries.is_empty() {
+                return None;
+            }
+            if let Some(found) = self.scan_unhashed(key) {
+                return found;
+            }
+        }
+        self.find_hashed(key.probe_hash(), key).ok()
+    }
+
+    /// A small table's lookup without hashing the probe (see
+    /// [`Probe::probe_eq_unhashed`]); `None` when some entry needs the
+    /// hash.
+    #[inline(always)]
+    fn scan_unhashed<Q: Probe + ?Sized>(&self, key: &Q) -> Option<Option<usize>> {
+        let mut found = None;
+        for (i, b) in self.entries.iter().enumerate() {
+            match key.probe_eq_unhashed(&b.key)? {
+                true => {
+                    found = Some(i);
+                    break;
+                }
+                false => {}
+            }
+        }
+        Some(found)
+    }
+
+    #[inline]
+    pub fn get<Q: Probe + ?Sized>(&self, key: &Q) -> Option<&Object> {
+        let i = self.get_index_of(key)?;
+        // SAFETY: a found position is in bounds.
+        Some(unsafe { &self.entries.get_unchecked(i).value })
+    }
+
+    #[inline]
+    pub fn get_mut<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<&mut Object> {
+        let i = self.get_index_of(key)?;
+        // SAFETY: a found position is in bounds.
+        Some(unsafe { &mut self.entries.get_unchecked_mut(i).value })
+    }
+
+    #[inline]
+    pub fn get_full<Q: Probe + ?Sized>(&self, key: &Q) -> Option<(usize, &DictKey, &Object)> {
+        let i = self.get_index_of(key)?;
+        let b = &self.entries[i];
+        Some((i, &b.key, &b.value))
+    }
+
+    #[inline]
+    pub fn get_full_mut<Q: Probe + ?Sized>(
+        &mut self,
+        key: &Q,
+    ) -> Option<(usize, &DictKey, &mut Object)> {
+        let i = self.get_index_of(key)?;
+        let b = &mut self.entries[i];
+        Some((i, &b.key, &mut b.value))
+    }
+
+    #[inline]
+    pub fn get_key_value<Q: Probe + ?Sized>(&self, key: &Q) -> Option<(&DictKey, &Object)> {
+        let i = self.get_index_of(key)?;
+        let b = &self.entries[i];
+        Some((&b.key, &b.value))
+    }
+
+    #[inline]
+    pub fn contains_key<Q: Probe + ?Sized>(&self, key: &Q) -> bool {
+        self.get_index_of(key).is_some()
+    }
+
+    /// The position of the first entry with Python hash `hash` that `eq`
+    /// accepts; `eq` sees only keys of that exact hash.
+    pub fn find_by_hash(&self, hash: i64, mut eq: impl FnMut(&DictKey) -> bool) -> Option<usize> {
+        struct Pred<'a, F>(std::cell::RefCell<&'a mut F>);
+        impl<F: FnMut(&DictKey) -> bool> Probe for Pred<'_, F> {
+            fn probe_hash(&self) -> i64 {
+                unreachable!("hashed by the caller")
+            }
+            fn probe_eq(&self, stored: &DictKey) -> bool {
+                (self.0.borrow_mut())(stored)
+            }
+        }
+        self.find_hashed(hash, &Pred(std::cell::RefCell::new(&mut eq)))
+            .ok()
+    }
+
+    /// Every stored key with Python hash `hash`, in probe order.
+    pub fn keys_with_hash(&self, hash: i64) -> Vec<&DictKey> {
+        let mut out = Vec::new();
+        let _ = self.find_by_hash(hash, |k| {
+            out.push(k as *const DictKey);
+            false
+        });
+        // SAFETY: the pointers address entries of `self`, borrowed for the
+        // returned lifetime.
+        out.into_iter().map(|p| unsafe { &*p }).collect()
+    }
+
+    /// The stored Python hash of the entry at `i`.
+    #[inline]
+    pub fn hash_at(&self, i: usize) -> Option<i64> {
+        self.entries.get(i).map(|b| b.hash)
+    }
+
+    // ----------------------------------------------------------------
+    // Positional access.
+
+    #[inline]
+    pub fn get_index(&self, i: usize) -> Option<(&DictKey, &Object)> {
+        self.entries.get(i).map(|b| (&b.key, &b.value))
+    }
+
+    #[inline]
+    pub fn get_index_mut(&mut self, i: usize) -> Option<(&DictKey, &mut Object)> {
+        self.entries.get_mut(i).map(|b| (&b.key, &mut b.value))
+    }
+
+    /// Mutable access to a key in place. The caller must keep its hash
+    /// and equality unchanged.
+    #[inline]
+    pub fn get_index_mut2(&mut self, i: usize) -> Option<(&mut DictKey, &mut Object)> {
+        self.entries.get_mut(i).map(|b| (&mut b.key, &mut b.value))
+    }
+
+    #[inline]
+    pub fn first(&self) -> Option<(&DictKey, &Object)> {
+        self.get_index(0)
+    }
+
+    #[inline]
+    pub fn last(&self) -> Option<(&DictKey, &Object)> {
+        self.entries.last().map(|b| (&b.key, &b.value))
+    }
+
+    // ----------------------------------------------------------------
+    // Insertion.
+
+    /// Append `key` (hash `hash`), known to be absent, at slot `pos` (from
+    /// a missed [`Self::find_hashed`]): its position.
+    #[inline]
+    fn push_at(&mut self, pos: usize, hash: i64, key: DictKey, value: Object) -> usize {
+        let i = self.entries.len();
+        if self.slots.is_empty() {
+            if i < SMALL {
+                self.entries.push(Bucket { hash, key, value });
+                return i;
+            }
+            self.entries.push(Bucket { hash, key, value });
+            self.rebuild_slots(slots_for(i + 1));
+            return i;
+        }
+        if (i + 1) * 3 > self.slots.len() * 2 {
+            self.entries.push(Bucket { hash, key, value });
+            self.rebuild_slots(self.slots.len() * 2);
+            return i;
+        }
+        assert!(i < IX_MASK as usize - 1, "dict too large");
+        // SAFETY: `pos` is an empty slot inside the table.
+        unsafe {
+            *self.slots.get_unchecked_mut(pos) = (mix(hash) & !IX_MASK) | (i as u64 + 1);
+        }
+        self.entries.push(Bucket { hash, key, value });
+        i
+    }
+
+    /// Insert or replace: the entry's position and the replaced value.
+    #[inline]
+    pub fn insert_full(&mut self, key: DictKey, value: Object) -> (usize, Option<Object>) {
+        let hash = key.probe_hash();
+        self.insert_hashed(hash, key, value)
+    }
+
+    /// [`Self::insert_full`] with the key's Python hash precomputed.
+    #[inline]
+    pub fn insert_hashed(&mut self, hash: i64, key: DictKey, value: Object) -> (usize, Option<Object>) {
+        match self.find_hashed(hash, &key) {
+            Ok(i) => {
+                // SAFETY: a found position is in bounds.
+                let slot = unsafe { &mut self.entries.get_unchecked_mut(i).value };
+                (i, Some(std::mem::replace(slot, value)))
+            }
+            Err(pos) => (self.push_at(pos, hash, key, value), None),
+        }
+    }
+
+    #[inline]
+    pub fn insert(&mut self, key: DictKey, value: Object) -> Option<Object> {
+        self.insert_full(key, value).1
+    }
+
+    /// Append `key` with Python hash `hash` without looking for an equal
+    /// key: the caller knows there is none. Its position.
+    pub fn insert_unique_hashed(&mut self, hash: i64, key: DictKey, value: Object) -> usize {
+        if self.slots.is_empty() {
+            return self.push_at(0, hash, key, value);
+        }
+        let m = mix(hash);
+        let mask = self.slots.len() - 1;
+        let mut pos = (m >> self.shift) as usize;
+        while self.slots[pos] != 0 {
+            pos = (pos + 1) & mask;
+        }
+        self.push_at(pos, hash, key, value)
+    }
+
+    /// The entry for `key`, to read, change, or fill.
+    pub fn entry(&mut self, key: DictKey) -> Entry<'_> {
+        let hash = key.probe_hash();
+        match self.find_hashed(hash, &key) {
+            Ok(index) => Entry::Occupied(OccupiedEntry { map: self, index }),
+            Err(pos) => Entry::Vacant(VacantEntry {
+                map: self,
+                key,
+                hash,
+                pos,
+            }),
+        }
+    }
+
+    /// The entry for a probe: its position if present, else a vacancy
+    /// that inserts the key the caller builds.
+    pub fn probe_entry<Q: Probe + ?Sized>(&mut self, key: &Q) -> ProbeEntry<'_> {
+        let hash = key.probe_hash();
+        match self.find_hashed(hash, key) {
+            Ok(index) => ProbeEntry::Occupied(OccupiedEntry { map: self, index }),
+            Err(pos) => ProbeEntry::Vacant(ProbeVacancy {
+                map: self,
+                hash,
+                pos,
+            }),
+        }
+    }
+
+    // ----------------------------------------------------------------
+    // Removal.
+
+    /// Remove entry `i`, shifting the later entries down.
+    pub fn shift_remove_index(&mut self, i: usize) -> Option<(DictKey, Object)> {
+        let n = self.entries.len();
+        if i >= n {
+            return None;
+        }
+        if self.slots.is_empty() {
+            let b = self.entries.remove(i);
+            return Some((b.key, b.value));
+        }
+        let pos = self.slot_of(self.entries[i].hash, i);
+        self.erase_slot(pos);
+        let b = self.entries.remove(i);
+        // Renumber the entries that moved down: one slot walk each when
+        // few moved, else one pass over the table.
+        let moved = n - 1 - i;
+        if moved * 4 <= self.slots.len() {
+            for j in i..n - 1 {
+                let h = self.entries[j].hash;
+                self.repoint(h, j + 1, j);
+            }
+        } else {
+            let above = i as u64 + 1;
+            for s in self.slots.iter_mut() {
+                if *s & IX_MASK > above {
+                    *s -= 1;
+                }
+            }
+        }
+        Some((b.key, b.value))
+    }
+
+    /// Remove entry `i`, moving the last entry into its place.
+    pub fn swap_remove_index(&mut self, i: usize) -> Option<(DictKey, Object)> {
+        let n = self.entries.len();
+        if i >= n {
+            return None;
+        }
+        if !self.slots.is_empty() {
+            let pos = self.slot_of(self.entries[i].hash, i);
+            self.erase_slot(pos);
+            if i != n - 1 {
+                let h = self.entries[n - 1].hash;
+                self.repoint(h, n - 1, i);
+            }
+        }
+        let b = self.entries.swap_remove(i);
+        Some((b.key, b.value))
+    }
+
+    /// Remove the last entry.
+    pub fn pop(&mut self) -> Option<(DictKey, Object)> {
+        let n = self.entries.len();
+        if n == 0 {
+            return None;
+        }
+        if !self.slots.is_empty() {
+            let pos = self.slot_of(self.entries[n - 1].hash, n - 1);
+            self.erase_slot(pos);
+        }
+        let b = self.entries.pop()?;
+        Some((b.key, b.value))
+    }
+
+    pub fn shift_remove_full<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(usize, DictKey, Object)> {
+        let i = self.get_index_of(key)?;
+        let (k, v) = self.shift_remove_index(i)?;
+        Some((i, k, v))
+    }
+
+    pub fn shift_remove_entry<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(DictKey, Object)> {
+        let i = self.get_index_of(key)?;
+        self.shift_remove_index(i)
+    }
+
+    pub fn shift_remove<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<Object> {
+        self.shift_remove_entry(key).map(|(_, v)| v)
+    }
+
+    pub fn swap_remove<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<Object> {
+        let i = self.get_index_of(key)?;
+        self.swap_remove_index(i).map(|(_, v)| v)
+    }
+
+    pub fn swap_remove_full<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(usize, DictKey, Object)> {
+        let i = self.get_index_of(key)?;
+        let (k, v) = self.swap_remove_index(i)?;
+        Some((i, k, v))
+    }
+
+    /// Keep the entries `keep` accepts, in order.
+    pub fn retain(&mut self, mut keep: impl FnMut(&DictKey, &mut Object) -> bool) {
+        let n = self.entries.len();
+        self.entries.retain_mut(|b| keep(&b.key, &mut b.value));
+        if self.entries.len() != n {
+            self.reindex();
+        }
+    }
+
+    /// Keep the first `len` entries.
+    pub fn truncate(&mut self, len: usize) {
+        if len < self.entries.len() {
+            self.entries.truncate(len);
+            self.reindex();
+        }
+    }
+
+    /// Split off the entries from `at` on.
+    pub fn split_off(&mut self, at: usize) -> Self {
+        let tail = self.entries.split_off(at);
+        self.reindex();
+        let mut out = Self {
+            entries: tail,
+            slots: Box::new([]),
+            shift: 64,
+        };
+        if out.entries.len() > SMALL {
+            out.rebuild_slots(slots_for(out.entries.len()));
+        }
+        out
+    }
+
+    /// Remove the entries in `range`, yielding them.
+    pub fn drain<R: std::ops::RangeBounds<usize>>(&mut self, range: R) -> std::vec::IntoIter<(DictKey, Object)> {
+        let out: Vec<(DictKey, Object)> = self.entries.drain(range).map(|b| (b.key, b.value)).collect();
+        self.reindex();
+        out.into_iter()
+    }
+
+    /// Move entry `from` to position `to`, shifting the ones between.
+    pub fn move_index(&mut self, from: usize, to: usize) {
+        let n = self.entries.len();
+        assert!(from < n && to < n, "move_index out of bounds");
+        if from == to {
+            return;
+        }
+        if self.slots.is_empty() {
+            if from < to {
+                self.entries[from..=to].rotate_left(1);
+            } else {
+                self.entries[to..=from].rotate_right(1);
+            }
+            return;
+        }
+        let (lo, hi) = (from.min(to), from.max(to));
+        // Locate every affected slot before renumbering any.
+        let at: Vec<usize> = (lo..=hi)
+            .map(|k| self.slot_of(self.entries[k].hash, k))
+            .collect();
+        for (k, &pos) in (lo..=hi).zip(&at) {
+            let new = if k == from {
+                to
+            } else if from < to {
+                k - 1
+            } else {
+                k + 1
+            };
+            self.slots[pos] = (self.slots[pos] & !IX_MASK) | (new as u64 + 1);
+        }
+        if from < to {
+            self.entries[from..=to].rotate_left(1);
+        } else {
+            self.entries[to..=from].rotate_right(1);
+        }
+    }
+
+    /// Swap the entries at `a` and `b`.
+    pub fn swap_indices(&mut self, a: usize, b: usize) {
+        if a == b {
+            return;
+        }
+        if !self.slots.is_empty() {
+            let pa = self.slot_of(self.entries[a].hash, a);
+            let pb = self.slot_of(self.entries[b].hash, b);
+            self.slots[pa] = (self.slots[pa] & !IX_MASK) | (b as u64 + 1);
+            self.slots[pb] = (self.slots[pb] & !IX_MASK) | (a as u64 + 1);
+        }
+        self.entries.swap(a, b);
+    }
+
+    /// Reverse the order.
+    pub fn reverse(&mut self) {
+        self.entries.reverse();
+        self.reindex();
+    }
+
+    /// Sort the entries with `cmp` (stable).
+    pub fn sort_by(&mut self, mut cmp: impl FnMut(&DictKey, &Object, &DictKey, &Object) -> std::cmp::Ordering) {
+        self.entries
+            .sort_by(|a, b| cmp(&a.key, &a.value, &b.key, &b.value));
+        self.reindex();
+    }
+
+    /// Sort the entries by key with `cmp` (stable).
+    pub fn sort_by_key_with(&mut self, mut cmp: impl FnMut(&DictKey, &DictKey) -> std::cmp::Ordering) {
+        self.entries.sort_by(|a, b| cmp(&a.key, &b.key));
+        self.reindex();
+    }
+
+    // ----------------------------------------------------------------
+    // Iteration.
+
+    #[inline]
+    pub fn iter(&self) -> Iter<'_> {
+        Iter(self.entries.iter())
+    }
+
+    #[inline]
+    pub fn iter_mut(&mut self) -> IterMut<'_> {
+        IterMut(self.entries.iter_mut())
+    }
+
+    /// Mutable keys and values. The caller must keep every key's hash and
+    /// equality unchanged.
+    #[inline]
+    pub fn iter_mut2(&mut self) -> IterMut2<'_> {
+        IterMut2(self.entries.iter_mut())
+    }
+
+    #[inline]
+    pub fn keys(&self) -> Keys<'_> {
+        Keys(self.entries.iter())
+    }
+
+    #[inline]
+    pub fn values(&self) -> Values<'_> {
+        Values(self.entries.iter())
+    }
+
+    #[inline]
+    pub fn values_mut(&mut self) -> ValuesMut<'_> {
+        ValuesMut(self.entries.iter_mut())
+    }
+
+    pub fn into_keys(self) -> impl DoubleEndedIterator<Item = DictKey> + ExactSizeIterator {
+        self.entries.into_iter().map(|b| b.key)
+    }
+
+    pub fn into_values(self) -> impl DoubleEndedIterator<Item = Object> + ExactSizeIterator {
+        self.entries.into_iter().map(|b| b.value)
+    }
+}
+
+/// An entry of [`DictMap::entry`].
+#[derive(Debug)]
+pub enum Entry<'a> {
+    Occupied(OccupiedEntry<'a>),
+    Vacant(VacantEntry<'a>),
+}
+
+/// A present entry.
+#[derive(Debug)]
+pub struct OccupiedEntry<'a> {
+    map: &'a mut DictMap,
+    index: usize,
+}
+
+/// An absent key, with the slot it would take.
+#[derive(Debug)]
+pub struct VacantEntry<'a> {
+    map: &'a mut DictMap,
+    key: DictKey,
+    hash: i64,
+    pos: usize,
+}
+
+/// An entry of [`DictMap::probe_entry`].
+#[derive(Debug)]
+pub enum ProbeEntry<'a> {
+    Occupied(OccupiedEntry<'a>),
+    Vacant(ProbeVacancy<'a>),
+}
+
+/// An absent probe key, with its hash and the slot it would take.
+#[derive(Debug)]
+pub struct ProbeVacancy<'a> {
+    map: &'a mut DictMap,
+    hash: i64,
+    pos: usize,
+}
+
+impl<'a> Entry<'a> {
+    pub fn or_insert(self, default: Object) -> &'a mut Object {
+        match self {
+            Entry::Occupied(e) => e.into_mut(),
+            Entry::Vacant(e) => e.insert(default),
+        }
+    }
+
+    pub fn or_insert_with(self, f: impl FnOnce() -> Object) -> &'a mut Object {
+        match self {
+            Entry::Occupied(e) => e.into_mut(),
+            Entry::Vacant(e) => e.insert(f()),
+        }
+    }
+
+    pub fn index(&self) -> usize {
+        match self {
+            Entry::Occupied(e) => e.index,
+            Entry::Vacant(e) => e.map.len(),
+        }
+    }
+}
+
+impl<'a> OccupiedEntry<'a> {
+    #[inline]
+    pub fn index(&self) -> usize {
+        self.index
+    }
+
+    #[inline]
+    pub fn key(&self) -> &DictKey {
+        &self.map.entries[self.index].key
+    }
+
+    #[inline]
+    pub fn get(&self) -> &Object {
+        &self.map.entries[self.index].value
+    }
+
+    #[inline]
+    pub fn get_mut(&mut self) -> &mut Object {
+        &mut self.map.entries[self.index].value
+    }
+
+    #[inline]
+    pub fn into_mut(self) -> &'a mut Object {
+        &mut self.map.entries[self.index].value
+    }
+
+    #[inline]
+    pub fn insert(&mut self, value: Object) -> Object {
+        std::mem::replace(self.get_mut(), value)
+    }
+
+    pub fn shift_remove_entry(self) -> (DictKey, Object) {
+        self.map
+            .shift_remove_index(self.index)
+            .expect("occupied entry is present")
+    }
+
+    pub fn shift_remove(self) -> Object {
+        self.shift_remove_entry().1
+    }
+}
+
+impl<'a> VacantEntry<'a> {
+    #[inline]
+    pub fn key(&self) -> &DictKey {
+        &self.key
+    }
+
+    pub fn insert(self, value: Object) -> &'a mut Object {
+        let i = self.map.push_at(self.pos, self.hash, self.key, value);
+        &mut self.map.entries[i].value
+    }
+}
+
+impl<'a> ProbeVacancy<'a> {
+    /// The probe's Python hash.
+    #[inline]
+    pub fn hash(&self) -> i64 {
+        self.hash
+    }
+
+    /// Insert `key`, which must equal the probe (same hash, same key):
+    /// its position.
+    #[inline]
+    pub fn insert(self, key: DictKey, value: Object) -> usize {
+        self.map.push_at(self.pos, self.hash, key, value)
+    }
+}
+
+macro_rules! entry_iter {
+    ($name:ident, $inner:ty, $item:ty, |$b:ident| $map:expr) => {
+        pub struct $name<'a>($inner);
+
+        impl<'a> Iterator for $name<'a> {
+            type Item = $item;
+
+            #[inline]
+            fn next(&mut self) -> Option<Self::Item> {
+                self.0.next().map(|$b| $map)
+            }
+
+            #[inline]
+            fn size_hint(&self) -> (usize, Option<usize>) {
+                self.0.size_hint()
+            }
+
+            #[inline]
+            fn nth(&mut self, n: usize) -> Option<Self::Item> {
+                self.0.nth(n).map(|$b| $map)
+            }
+
+            #[inline]
+            fn count(self) -> usize {
+                self.0.len()
+            }
+
+            #[inline]
+            fn last(mut self) -> Option<Self::Item> {
+                self.next_back()
+            }
+        }
+
+        impl DoubleEndedIterator for $name<'_> {
+            #[inline]
+            fn next_back(&mut self) -> Option<Self::Item> {
+                self.0.next_back().map(|$b| $map)
+            }
+
+            #[inline]
+            fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
+                self.0.nth_back(n).map(|$b| $map)
+            }
+        }
+
+        impl ExactSizeIterator for $name<'_> {
+            #[inline]
+            fn len(&self) -> usize {
+                self.0.len()
+            }
+        }
+
+        impl std::iter::FusedIterator for $name<'_> {}
+
+        impl fmt::Debug for $name<'_> {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                f.debug_struct(stringify!($name)).finish_non_exhaustive()
+            }
+        }
+    };
+}
+
+entry_iter!(Iter, std::slice::Iter<'a, Bucket>, (&'a DictKey, &'a Object), |b| (&b.key, &b.value));
+entry_iter!(IterMut, std::slice::IterMut<'a, Bucket>, (&'a DictKey, &'a mut Object), |b| (&b.key, &mut b.value));
+entry_iter!(IterMut2, std::slice::IterMut<'a, Bucket>, (&'a mut DictKey, &'a mut Object), |b| (&mut b.key, &mut b.value));
+entry_iter!(Keys, std::slice::Iter<'a, Bucket>, &'a DictKey, |b| &b.key);
+entry_iter!(Values, std::slice::Iter<'a, Bucket>, &'a Object, |b| &b.value);
+entry_iter!(ValuesMut, std::slice::IterMut<'a, Bucket>, &'a mut Object, |b| &mut b.value);
+
+impl Clone for Iter<'_> {
+    fn clone(&self) -> Self {
+        Self(self.0.clone())
+    }
+}
+
+impl Clone for Keys<'_> {
+    fn clone(&self) -> Self {
+        Self(self.0.clone())
+    }
+}
+
+impl Clone for Values<'_> {
+    fn clone(&self) -> Self {
+        Self(self.0.clone())
+    }
+}
+
+/// The owning iterator of a [`DictMap`].
+pub struct IntoIter(std::vec::IntoIter<Bucket>);
+
+impl Iterator for IntoIter {
+    type Item = (DictKey, Object);
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0.next().map(|b| (b.key, b.value))
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.0.size_hint()
+    }
+}
+
+impl DoubleEndedIterator for IntoIter {
+    #[inline]
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.0.next_back().map(|b| (b.key, b.value))
+    }
+}
+
+impl ExactSizeIterator for IntoIter {}
+
+impl fmt::Debug for IntoIter {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("IntoIter").finish_non_exhaustive()
+    }
+}
+
+impl IntoIterator for DictMap {
+    type Item = (DictKey, Object);
+    type IntoIter = IntoIter;
+
+    fn into_iter(self) -> IntoIter {
+        IntoIter(self.entries.into_iter())
+    }
+}
+
+impl<'a> IntoIterator for &'a DictMap {
+    type Item = (&'a DictKey, &'a Object);
+    type IntoIter = Iter<'a>;
+
+    fn into_iter(self) -> Iter<'a> {
+        self.iter()
+    }
+}
+
+impl<'a> IntoIterator for &'a mut DictMap {
+    type Item = (&'a DictKey, &'a mut Object);
+    type IntoIter = IterMut<'a>;
+
+    fn into_iter(self) -> IterMut<'a> {
+        self.iter_mut()
+    }
+}
+
+impl Extend<(DictKey, Object)> for DictMap {
+    fn extend<I: IntoIterator<Item = (DictKey, Object)>>(&mut self, iter: I) {
+        let iter = iter.into_iter();
+        let (lo, _) = iter.size_hint();
+        // (As IndexMap: reserve for all when empty, else half, since some
+        // keys may already be present.)
+        self.reserve(if self.is_empty() { lo } else { lo.div_ceil(2) });
+        for (k, v) in iter {
+            self.insert(k, v);
+        }
+    }
+}
+
+impl<'a> Extend<(&'a DictKey, &'a Object)> for DictMap {
+    fn extend<I: IntoIterator<Item = (&'a DictKey, &'a Object)>>(&mut self, iter: I) {
+        self.extend(iter.into_iter().map(|(k, v)| (k.clone(), v.clone())));
+    }
+}
+
+impl FromIterator<(DictKey, Object)> for DictMap {
+    fn from_iter<I: IntoIterator<Item = (DictKey, Object)>>(iter: I) -> Self {
+        let mut m = Self::new();
+        m.extend(iter);
+        m
+    }
+}
+
+impl<const N: usize> From<[(DictKey, Object); N]> for DictMap {
+    fn from(arr: [(DictKey, Object); N]) -> Self {
+        Self::from_iter(arr)
+    }
+}
+
+impl<Q: Probe + ?Sized> std::ops::Index<&Q> for DictMap {
+    type Output = Object;
+
+    fn index(&self, key: &Q) -> &Object {
+        self.get(key).expect("key not found")
+    }
+}
+
+impl std::ops::Index<usize> for DictMap {
+    type Output = Object;
+
+    fn index(&self, i: usize) -> &Object {
+        &self.entries[i].value
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn k(i: i64) -> DictKey {
+        DictKey(Object::Int(i))
+    }
+
+    fn check(m: &DictMap) {
+        for (i, b) in m.entries.iter().enumerate() {
+            assert_eq!(m.get_index_of(&b.key), Some(i));
+        }
+        if !m.slots.is_empty() {
+            let used = m.slots.iter().filter(|s| **s != 0).count();
+            assert_eq!(used, m.len());
+        }
+    }
+
+    #[test]
+    fn insert_remove_order() {
+        let mut m = DictMap::new();
+        for i in 0..200 {
+            assert_eq!(m.insert(k(i * 1024), Object::Int(i)), None);
+            check(&m);
+        }
+        for i in (0..200).step_by(3) {
+            assert!(m.shift_remove(&k(i * 1024)).is_some());
+            check(&m);
+        }
+        let keys: Vec<i64> = m
+            .keys()
+            .map(|k| match k.0 {
+                Object::Int(i) => i / 1024,
+                _ => unreachable!(),
+            })
+            .collect();
+        let want: Vec<i64> = (0..200).filter(|i| i % 3 != 0).collect();
+        assert_eq!(keys, want);
+        m.move_index(0, 50);
+        check(&m);
+        m.move_index(60, 3);
+        check(&m);
+        m.swap_remove_index(7);
+        check(&m);
+        while m.pop().is_some() {
+            check(&m);
+        }
+        assert!(m.is_empty());
+    }
+
+    #[test]
+    fn small_and_clone() {
+        let mut m = DictMap::new();
+        for i in 0..8 {
+            m.insert(k(i), Object::Int(i));
+        }
+        assert!(m.slots.is_empty());
+        m.insert(k(8), Object::Int(8));
+        assert!(!m.slots.is_empty());
+        let c = m.clone();
+        check(&c);
+        assert_eq!(c.get(&k(5)).map(|o| matches!(o, Object::Int(5))), Some(true));
+        assert_eq!(c.get(&k(50)).is_none(), true);
+    }
+}
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 22750282..f88c1769 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -2386,8 +2386,10 @@ unsafe fn store_subscr_fast(
                 let Some(d) = cell.peek_mut() else {
                     return 1;
                 };
-                let (old, changed) = match d.get_mut(&probe) {
-                    Some(slot) => {
+                // One probe finds the entry or the slot a new key takes.
+                let (old, changed) = match d.probe_entry(&probe) {
+                    crate::dictmap::ProbeEntry::Occupied(e) => {
+                        let slot = e.into_mut();
                         if !displaced_ok(slot) {
                             return 1;
                         }
@@ -2395,11 +2397,11 @@ unsafe fn store_subscr_fast(
                         let changed = !old.is_same(slot);
                         (old, changed)
                     }
-                    None => {
+                    crate::dictmap::ProbeEntry::Vacant(e) => {
                         if !probe.miss_is_exact() {
                             return 1;
                         }
-                        d.insert(crate::object::DictKey(crate::clone_hot(k)), value());
+                        e.insert(crate::object::DictKey(crate::clone_hot(k)), value());
                         (Object::None, true)
                     }
                 };
diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index f501881c..2515c78b 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -1123,7 +1123,6 @@ impl InstDict {
             DictData::with_capacity_and_hasher(n, crate::fasthash::FxBuildHasher)
         };
         if let Some(keys) = keys {
-            use indexmap::map::raw_entry_v1::{RawEntryApiV1, RawEntryMut};
             // A deferred owner holds only atomic values, and a tracked
             // one needs no barrier: insert without either. The names are
             // distinct `str`s whose hashes the table kept, so each goes
@@ -1132,13 +1131,8 @@ impl InstDict {
             for (i, v) in values.into_iter().enumerate() {
                 if let Some(k) = keys.get(i) {
                     // SAFETY: slot `i` is published (`get` found its name).
-                    let hash = crate::object::dict_table_hash(unsafe { *keys.hashes[i].get() });
-                    match map.raw_entry_mut_v1().from_hash(hash, |_| false) {
-                        RawEntryMut::Vacant(e) => {
-                            e.insert_hashed_nocheck(hash, k.clone(), v);
-                        }
-                        RawEntryMut::Occupied(_) => unreachable!("never matches"),
-                    }
+                    let hash = unsafe { *keys.hashes[i].get() };
+                    map.insert_unique_hashed(hash, k.clone(), v);
                 }
             }
         }
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index e7527cb5..ca50babe 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -47,6 +47,7 @@ pub mod builtins;
 pub mod capi_watchers;
 pub mod cpython_headers;
 pub mod descr_registry;
+pub mod dictmap;
 pub mod error;
 pub mod ext_loader;
 pub mod fasthash;
@@ -17609,8 +17610,9 @@ impl Interpreter {
                         // SAFETY: as above (the probe and the insert run
                         // no code).
                         let d = unsafe { cell.peek_mut() }?;
-                        let (old, changed) = match d.get_mut(&probe) {
-                            Some(slot) => {
+                        let (old, changed) = match d.probe_entry(&probe) {
+                            crate::dictmap::ProbeEntry::Occupied(e) => {
+                                let slot = e.into_mut();
                                 if !displaced_ok(slot) {
                                     return None;
                                 }
@@ -17620,12 +17622,12 @@ impl Interpreter {
                                 let changed = !old.is_same(slot);
                                 (old, changed)
                             }
-                            None => {
+                            crate::dictmap::ProbeEntry::Vacant(e) => {
                                 if !probe.miss_is_exact() {
                                     return None;
                                 }
                                 // SAFETY: as above.
-                                d.insert(DictKey(clone_hot(k)), unsafe {
+                                e.insert(DictKey(clone_hot(k)), unsafe {
                                     base.add(len - 3).read()
                                 });
                                 (Object::None, true)
@@ -17913,8 +17915,9 @@ impl Interpreter {
                         // SAFETY: as above.
                         let d = unsafe { cell.peek_mut() }?;
                         let (k, v) = (base.wrapping_add(len - 2), base.wrapping_add(len - 1));
-                        match d.get_mut(&probe) {
-                            Some(slot) => {
+                        match d.probe_entry(&probe) {
+                            crate::dictmap::ProbeEntry::Occupied(e) => {
+                                let slot = e.into_mut();
                                 if !Self::core_droppable(slot) || !Self::core_droppable(key) {
                                     return None;
                                 }
@@ -17926,11 +17929,11 @@ impl Interpreter {
                                     drop_hot(k.read());
                                 }
                             }
-                            None if probe.miss_is_exact() => {
+                            crate::dictmap::ProbeEntry::Vacant(e) if probe.miss_is_exact() => {
                                 // SAFETY: as above.
-                                unsafe { d.insert(DictKey(k.read()), v.read()) };
+                                unsafe { e.insert(DictKey(k.read()), v.read()) };
                             }
-                            None => return None,
+                            crate::dictmap::ProbeEntry::Vacant(_) => return None,
                         }
                     }
                     _ => return None,
@@ -21689,9 +21692,7 @@ impl Interpreter {
         // SAFETY: as above; the shared view above is no longer used.
         if let Some(d) = unsafe { m.dict.peek_mut() } {
             let map: &mut crate::object::DictMap = &mut *d;
-            if let Some((key, _)) =
-                indexmap::map::MutableKeys::get_index_mut2(map, key_idx as usize)
-            {
+            if let Some((key, _)) = map.get_index_mut2(key_idx as usize) {
                 key.0 = Object::Str(name);
             }
         }
@@ -23326,14 +23327,16 @@ impl Interpreter {
                                 break;
                             };
                             let value = stack[n - 1].clone();
-                            if let Some(slot) = d.get_mut(&probe) {
-                                drop(std::mem::replace(slot, value));
-                                true
-                            } else if probe.miss_is_exact() {
-                                d.insert(DictKey(key.clone()), value);
-                                true
-                            } else {
-                                false
+                            match d.probe_entry(&probe) {
+                                crate::dictmap::ProbeEntry::Occupied(e) => {
+                                    drop(std::mem::replace(e.into_mut(), value));
+                                    true
+                                }
+                                crate::dictmap::ProbeEntry::Vacant(e) if probe.miss_is_exact() => {
+                                    e.insert(DictKey(key.clone()), value);
+                                    true
+                                }
+                                crate::dictmap::ProbeEntry::Vacant(_) => false,
                             }
                         }
                         _ => false,
@@ -23714,19 +23717,20 @@ impl Interpreter {
                             let Ok(mut d) = cell.try_borrow_mut() else {
                                 break;
                             };
-                            let (old, changed) = match d.get_mut(&probe) {
-                                Some(slot) => {
+                            let (old, changed) = match d.probe_entry(&probe) {
+                                crate::dictmap::ProbeEntry::Occupied(e) => {
+                                    let slot = e.into_mut();
                                     let v = std::mem::replace(value_slot, Object::Unbound);
                                     let old = std::mem::replace(slot, v);
                                     let changed = !old.is_same(slot);
                                     (old, changed)
                                 }
-                                None => {
+                                crate::dictmap::ProbeEntry::Vacant(e) => {
                                     if !probe.miss_is_exact() {
                                         break;
                                     }
                                     let v = std::mem::replace(value_slot, Object::Unbound);
-                                    d.insert(DictKey(key.clone()), v);
+                                    e.insert(DictKey(key.clone()), v);
                                     (Object::Unbound, true)
                                 }
                             };
@@ -45769,7 +45773,7 @@ impl Interpreter {
                 if let Some((k, v)) = g.get_index(key_idx as usize) {
                     // Verify the key at the cached slot still matches the
                     // expected name. `del` of an earlier global shift-removes
-                    // an IndexMap entry, renumbering every later slot without
+                    // a table entry, renumbering every later slot without
                     // changing the dict's Rc identity — so the cached index
                     // would otherwise alias a *different* global's value.
                     if let Object::Str(s) = &k.0 {
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 76d0d507..97d6b4e3 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -2390,7 +2390,7 @@ impl DictViewKind {
 }
 
 /// Live view over a backing `dict`. Iteration order mirrors the
-/// underlying `IndexMap`. Mutations to the dict propagate through
+/// underlying table. Mutations to the dict propagate through
 /// the view (CPython invariant).
 #[derive(Debug)]
 pub struct PyDictView {
@@ -3280,7 +3280,8 @@ pub fn module_class(m: &Rc<PyModule>) -> Option<Rc<crate::types::TypeObject>> {
 }
 
 /// Dictionary key: a hashable [`Object`] wrapped to satisfy the
-/// `Hash + Eq` requirements imposed by `HashMap` / `IndexMap`.
+/// `Hash + Eq` requirements of `HashMap` / `IndexSet`, and the stored key
+/// type of a dict's [`crate::dictmap::DictMap`].
 #[derive(Clone, Debug)]
 pub struct DictKey(pub Object);
 
@@ -3305,21 +3306,36 @@ pub struct DictKey(pub Object);
 #[derive(Debug, Clone, Copy)]
 pub struct StrKey<'a>(pub &'a str);
 
-impl Hash for StrKey<'_> {
+impl crate::dictmap::Probe for StrKey<'_> {
     #[inline]
-    fn hash<H: Hasher>(&self, state: &mut H) {
+    fn probe_hash(&self) -> i64 {
         // Bit-for-bit the `DictKey` hash of `Object::Str(self.0)`.
-        py_str_hash(self.0).hash(state);
+        py_str_hash(self.0)
     }
-}
 
-impl indexmap::Equivalent<DictKey> for StrKey<'_> {
     #[inline]
-    fn equivalent(&self, key: &DictKey) -> bool {
+    fn probe_eq_unhashed(&self, key: &DictKey) -> Option<bool> {
+        // Comparing a name with a small table's stored strings costs less
+        // than hashing it.
+        match &key.0 {
+            Object::Str(s) => Some(
+                std::ptr::eq(s.as_ptr(), self.0.as_ptr())
+                    || crate::dictmap::key_bytes_eq(s.as_bytes(), self.0.as_bytes()),
+            ),
+            Object::WStr(_) => Some(false),
+            _ => None,
+        }
+    }
+
+    #[inline]
+    fn probe_eq(&self, key: &DictKey) -> bool {
         match &key.0 {
             // Interned names probe with the stored key's own bytes: the
             // pointer settles it before any byte compare.
-            Object::Str(s) => std::ptr::eq(s.as_ptr(), self.0.as_ptr()) || &**s == self.0,
+            Object::Str(s) => {
+                std::ptr::eq(s.as_ptr(), self.0.as_ptr())
+                    || crate::dictmap::key_bytes_eq(s.as_bytes(), self.0.as_bytes())
+            }
             // A `WStr` always carries a lone surrogate (module invariant),
             // so it can never equal a valid `&str`.
             Object::WStr(_) => false,
@@ -3345,17 +3361,15 @@ pub struct StrKeyHashed<'a> {
     pub hash: i64,
 }
 
-impl Hash for StrKeyHashed<'_> {
+impl crate::dictmap::Probe for StrKeyHashed<'_> {
     #[inline]
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.hash.hash(state);
+    fn probe_hash(&self) -> i64 {
+        self.hash
     }
-}
 
-impl indexmap::Equivalent<DictKey> for StrKeyHashed<'_> {
     #[inline]
-    fn equivalent(&self, key: &DictKey) -> bool {
-        indexmap::Equivalent::equivalent(&StrKey(self.s), key)
+    fn probe_eq(&self, key: &DictKey) -> bool {
+        crate::dictmap::Probe::probe_eq(&StrKey(self.s), key)
     }
 }
 
@@ -3388,18 +3402,19 @@ impl<'a> LeafNameProbe<'a> {
     }
 }
 
-impl Hash for LeafNameProbe<'_> {
+impl crate::dictmap::Probe for LeafNameProbe<'_> {
     #[inline]
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.hash.hash(state);
+    fn probe_hash(&self) -> i64 {
+        self.hash
     }
-}
 
-impl indexmap::Equivalent<DictKey> for LeafNameProbe<'_> {
     #[inline]
-    fn equivalent(&self, key: &DictKey) -> bool {
+    fn probe_eq(&self, key: &DictKey) -> bool {
         match &key.0 {
-            Object::Str(k) => std::ptr::eq(k.as_ptr(), self.s.as_ptr()) || &**k == self.s,
+            Object::Str(k) => {
+                std::ptr::eq(k.as_ptr(), self.s.as_ptr())
+                    || crate::dictmap::key_bytes_eq(k.as_bytes(), self.s.as_bytes())
+            }
             // A lone surrogate never equals a valid `&str`.
             Object::WStr(_) => false,
             _ => {
@@ -3431,11 +3446,24 @@ impl<'a> LeafProbe<'a> {
     /// metaclass keeps `type`'s `__hash__` and `__eq__`, a function,
     /// generator-family object or module, or a tuple of machine ints and
     /// strings.
-    #[inline]
+    #[inline(always)]
     pub fn new(key: &'a Object) -> Option<Self> {
         let hash = match key {
             Object::Str(s) => SharedStr::hash_cached(s),
             Object::Int(i) => py_hash_long_i64(*i),
+            _ => Self::other_hash(key)?,
+        };
+        Some(Self {
+            key,
+            hash,
+            foreign: std::cell::Cell::new(false),
+        })
+    }
+
+    /// [`Self::new`]'s hash for a key other than a `str` or an `int`.
+    #[inline(never)]
+    fn other_hash(key: &Object) -> Option<i64> {
+        Some(match key {
             Object::Instance(i)
                 if i.native.get().is_none()
                     && i.class_dunder(crate::types::Dunder::Hash).object_owner() =>
@@ -3458,11 +3486,6 @@ impl<'a> LeafProbe<'a> {
             | Object::AsyncGenerator(_)
             | Object::Module(_) => identity_hash(key),
             _ => return None,
-        };
-        Some(Self {
-            key,
-            hash,
-            foreign: std::cell::Cell::new(false),
         })
     }
 
@@ -3514,13 +3537,36 @@ impl Hash for LeafProbe<'_> {
 impl indexmap::Equivalent<DictKey> for LeafProbe<'_> {
     #[inline]
     fn equivalent(&self, key: &DictKey) -> bool {
+        crate::dictmap::Probe::probe_eq(self, key)
+    }
+}
+
+impl crate::dictmap::Probe for LeafProbe<'_> {
+    #[inline(always)]
+    fn probe_hash(&self) -> i64 {
+        self.hash
+    }
+
+    #[inline(always)]
+    fn probe_eq(&self, key: &DictKey) -> bool {
         match (self.key, &key.0) {
             // (The same string object, an interned name's usual case,
             // settles before any byte compare.)
             (Object::Str(a), Object::Str(b)) => {
-                std::ptr::eq(a.as_ptr(), b.as_ptr()) || a.as_bytes() == b.as_bytes()
+                SharedStr::ptr_eq(a, b) || crate::dictmap::key_bytes_eq(a.as_bytes(), b.as_bytes())
             }
             (Object::Int(a), Object::Int(b)) => a == b,
+            _ => self.other_eq(key),
+        }
+    }
+}
+
+impl LeafProbe<'_> {
+    /// [`crate::dictmap::Probe::probe_eq`] for any pairing but two `str`s
+    /// or two `int`s.
+    #[inline(never)]
+    fn other_eq(&self, key: &DictKey) -> bool {
+        match (self.key, &key.0) {
             // Identity settles an instance probe (CPython compares keys
             // with `is` first); any other pairing may need `__eq__`.
             (Object::Instance(a), Object::Instance(b)) if Rc::ptr_eq(a, b) => true,
@@ -3885,7 +3931,9 @@ impl PartialEq for DictKey {
         // Two plain strings or machine ints, the common keys, are equal by
         // value alone.
         match (&self.0, &other.0) {
-            (Object::Str(a), Object::Str(b)) => return a.as_bytes() == b.as_bytes(),
+            (Object::Str(a), Object::Str(b)) => {
+                return crate::dictmap::key_bytes_eq(a.as_bytes(), b.as_bytes());
+            }
             (Object::Int(a), Object::Int(b)) => return a == b,
             _ => {}
         }
@@ -3920,7 +3968,8 @@ impl PartialEq for DictKey {
             }
             // CPython dispatches `__eq__` only between keys in the *same
             // hash bucket*; unequal hashes mean "not equal" without any
-            // user code. indexmap's small-table linear scan compares every
+            // user code. A dict's table compares stored hashes first, but
+            // a set's (`indexmap`'s) small-table linear scan compares every
             // entry through this `eq` with no hash check at all, so enforce
             // the bucket precondition here — sqlalchemy's
             // `constraints.discard(ColumnSet())` probes a 1-element set
@@ -4100,17 +4149,10 @@ pub(crate) fn dict_key_is_reentrant(key: &Object) -> bool {
     }
 }
 
-/// The table-level (Fx-mixed) hash for a Python hash value, matching what
-/// `DictKey::hash` feeds the [`DictData`] hasher.
-pub(crate) fn dict_table_hash(py_hash: i64) -> u64 {
-    use std::hash::BuildHasher;
-    crate::fasthash::FxBuildHasher.hash_one(py_hash)
-}
-
-/// Probe result of [`dict_reentrant_probe`]: the key's table-level hash
-/// plus the equal *stored* key, if any.
+/// Probe result of [`dict_reentrant_probe`]: the key's Python hash plus
+/// the equal *stored* key, if any.
 pub(crate) struct ReentrantProbe {
-    table_hash: u64,
+    hash: i64,
     stored: Option<Object>,
 }
 
@@ -4129,9 +4171,7 @@ pub(crate) fn dict_reentrant_probe(
     d: &RefCell<DictData>,
     key: &Object,
 ) -> Result<ReentrantProbe, RuntimeError> {
-    use indexmap::map::raw_entry_v1::RawEntryApiV1;
     let kh = key_cmp_scope(|| py_hash_value(key))?.unwrap_or_else(|| identity_hash(key));
-    let table_hash = dict_table_hash(kh);
     let mut memo: Vec<(Object, bool)> = Vec::new();
     loop {
         // Same-hash candidates, collected under a short borrow with a
@@ -4139,7 +4179,7 @@ pub(crate) fn dict_reentrant_probe(
         let mut cands: Vec<Object> = Vec::new();
         {
             let m = d.borrow();
-            let _ = m.raw_entry_v1().from_hash(table_hash, |k| {
+            let _ = m.find_by_hash(kh, |k| {
                 cands.push(k.0.clone());
                 false
             });
@@ -4164,19 +4204,8 @@ pub(crate) fn dict_reentrant_probe(
                 }
                 continue;
             }
-            // Same-bucket precondition: indexmap's small-table linear scan
-            // hands `from_hash` every entry regardless of hash, but CPython
-            // only ever dispatches `__eq__` between keys whose hashes match.
-            // The stored key's hash comes from its record (CPython reads the
-            // entry's `me_hash`); only a record-less key pays a dispatch.
-            let sh = match recorded_key_hash(stored) {
-                Some(h) => h,
-                None => key_cmp_scope(|| py_hash_value(stored))?
-                    .unwrap_or_else(|| identity_hash(stored)),
-            };
-            if sh != kh {
-                continue;
-            }
+            // (The table hands over only keys whose stored hash equals the
+            // probe's, CPython's `me_hash` check, so no hash is recomputed.)
             // Python `__eq__` (stored first, CPython's argument order) with
             // no borrow held; it may mutate `d` re-entrantly.
             let eq = key_cmp_scope(|| current_interp_eq(stored, key))?.unwrap_or(false);
@@ -4186,7 +4215,7 @@ pub(crate) fn dict_reentrant_probe(
         }
         if found.is_some() || !fresh_call {
             return Ok(ReentrantProbe {
-                table_hash,
+                hash: kh,
                 stored: found,
             });
         }
@@ -4211,25 +4240,22 @@ pub(crate) fn dict_reentrant_insert(
     key: Object,
     value: Object,
 ) -> Result<Option<Object>, RuntimeError> {
-    use indexmap::map::raw_entry_v1::{RawEntryApiV1, RawEntryMut};
     let probe = dict_reentrant_probe(d, &key)?;
     let target = probe.stored.as_ref().unwrap_or(&key);
     let mut m = d.borrow_mut();
-    match m
-        .raw_entry_mut_v1()
-        .from_hash(probe.table_hash, |k| k.0.is_same(target))
-    {
-        RawEntryMut::Occupied(mut e) => {
-            let same = e.get().is_same(&value);
-            let old = e.insert(value);
+    match m.find_by_hash(probe.hash, |k| k.0.is_same(target)) {
+        Some(i) => {
+            let (_, slot) = m.get_index_mut(i).expect("found above");
+            let same = slot.is_same(&value);
+            let old = std::mem::replace(slot, value);
             drop(m);
             if !same {
                 dict_mutation_event(d);
             }
             Ok(Some(old))
         }
-        RawEntryMut::Vacant(e) => {
-            e.insert_hashed_nocheck(probe.table_hash, DictKey(key), value);
+        None => {
+            m.insert_unique_hashed(probe.hash, DictKey(key), value);
             drop(m);
             dict_watch_bump(d);
             dict_mutation_event(d);
@@ -4246,17 +4272,13 @@ pub(crate) fn dict_reentrant_setdefault(
     key: Object,
     default: Object,
 ) -> Result<Object, RuntimeError> {
-    use indexmap::map::raw_entry_v1::{RawEntryApiV1, RawEntryMut};
     let probe = dict_reentrant_probe(d, &key)?;
     let target = probe.stored.as_ref().unwrap_or(&key);
     let mut m = d.borrow_mut();
-    match m
-        .raw_entry_mut_v1()
-        .from_hash(probe.table_hash, |k| k.0.is_same(target))
-    {
-        RawEntryMut::Occupied(e) => Ok(e.get().clone()),
-        RawEntryMut::Vacant(e) => {
-            e.insert_hashed_nocheck(probe.table_hash, DictKey(key), default.clone());
+    match m.find_by_hash(probe.hash, |k| k.0.is_same(target)) {
+        Some(i) => Ok(m.get_index(i).expect("found above").1.clone()),
+        None => {
+            m.insert_unique_hashed(probe.hash, DictKey(key), default.clone());
             drop(m);
             dict_watch_bump(d);
             dict_mutation_event(d);
@@ -4270,14 +4292,14 @@ pub(crate) fn dict_reentrant_get(
     d: &RefCell<DictData>,
     key: &Object,
 ) -> Result<Option<Object>, RuntimeError> {
-    use indexmap::map::raw_entry_v1::RawEntryApiV1;
     let probe = dict_reentrant_probe(d, key)?;
     match probe.stored {
-        Some(stored) => Ok(d
-            .borrow()
-            .raw_entry_v1()
-            .from_hash(probe.table_hash, |k| k.0.is_same(&stored))
-            .map(|(_, v)| v.clone())),
+        Some(stored) => {
+            let m = d.borrow();
+            Ok(m.find_by_hash(probe.hash, |k| k.0.is_same(&stored))
+                .and_then(|i| m.get_index(i))
+                .map(|(_, v)| v.clone()))
+        }
         None => Ok(None),
     }
 }
@@ -4289,13 +4311,11 @@ pub(crate) fn dict_index_of(
     d: &RefCell<DictData>,
     key: &Object,
 ) -> Result<Option<usize>, RuntimeError> {
-    use indexmap::map::raw_entry_v1::RawEntryApiV1;
     let probe = dict_reentrant_probe(d, key)?;
     match probe.stored {
         Some(stored) => Ok(d
             .borrow()
-            .raw_entry_v1()
-            .index_from_hash(probe.table_hash, |k| k.0.is_same(&stored))),
+            .find_by_hash(probe.hash, |k| k.0.is_same(&stored))),
         None => Ok(None),
     }
 }
@@ -4305,23 +4325,19 @@ pub(crate) fn dict_reentrant_remove(
     d: &RefCell<DictData>,
     key: &Object,
 ) -> Result<Option<(Object, Object)>, RuntimeError> {
-    use indexmap::map::raw_entry_v1::{RawEntryApiV1, RawEntryMut};
     let probe = dict_reentrant_probe(d, key)?;
     match probe.stored {
         Some(stored) => {
             let mut m = d.borrow_mut();
-            match m
-                .raw_entry_mut_v1()
-                .from_hash(probe.table_hash, |k| k.0.is_same(&stored))
-            {
-                RawEntryMut::Occupied(e) => {
-                    let (k, v) = e.shift_remove_entry();
+            match m.find_by_hash(probe.hash, |k| k.0.is_same(&stored)) {
+                Some(i) => {
+                    let (k, v) = m.shift_remove_index(i).expect("found above");
                     drop(m);
                     dict_watch_bump(d);
                     dict_mutation_event(d);
                     Ok(Some((k.0, v)))
                 }
-                RawEntryMut::Vacant(_) => Ok(None),
+                None => Ok(None),
             }
         }
         None => Ok(None),
@@ -4335,7 +4351,7 @@ pub(crate) fn dict_reentrant_remove(
 // same `ma_used`, so the size trip-wire stays silent — because the delete
 // leaves a tombstone and the reinsert *appends*: the iterator then finds an
 // entry beyond the `di->len` elements it expected and raises `RuntimeError:
-// dictionary keys changed during iteration`. Our `IndexMap` has no
+// dictionary keys changed during iteration`. Our table has no
 // tombstones (`shift_remove` compacts in place), so the layout carries no
 // trace of the churn. Instead, dict iterators register a *watch* on their
 // source dict; the centralized structural mutators (`dict_insert` on a new
@@ -4452,8 +4468,8 @@ pub(crate) fn dict_watch_bump(d: &RefCell<DictData>) {
 // `_testcapi.dict_get_version` exposes it and `test_dict_version` asserts
 // the full semantics.
 //
-// WeavePy's `DictData` is a bare `IndexMap` with no room for a per-dict
-// field, so versions live in a side registry keyed by the dict's heap
+// WeavePy's `DictData` has no room for a per-dict PEP 509 field, so
+// versions live in a side registry keyed by the dict's heap
 // address and assigned lazily: the first `dict_version_get` on a dict
 // registers it with a fresh tag; the centralized mutators
 // (`dict_insert`/`dict_remove`/`clear`/`popitem`/`setdefault` and the
@@ -4537,13 +4553,20 @@ impl Hash for DictKey {
         // (functions, types, plain instances, …) fold in their allocation
         // identity; truly unhashable keys share a constant bucket and the
         // runtime raises lazily when used.
-        let h = match &self.0 {
-            // (The commonest keys, without the general dispatch.)
-            Object::Str(s) => SharedStr::hash_cached(s),
-            Object::Int(i) => py_hash_long_i64(*i),
-            other => py_hash_value(other).unwrap_or_else(|| identity_hash(other)),
-        };
-        h.hash(state);
+        dict_key_hash(&self.0).hash(state);
+    }
+}
+
+/// The Python hash a dict or set stores for key `k`: its `hash()`, or its
+/// identity hash for an unhashable key (the runtime raises before such a
+/// key is stored).
+#[inline]
+pub(crate) fn dict_key_hash(k: &Object) -> i64 {
+    match k {
+        // (The commonest keys, without the general dispatch.)
+        Object::Str(s) => SharedStr::hash_cached(s),
+        Object::Int(i) => py_hash_long_i64(*i),
+        other => py_hash_value(other).unwrap_or_else(|| identity_hash(other)),
     }
 }
 
@@ -4555,7 +4578,7 @@ impl Hash for DictKey {
 /// (one multiply) instead of SipHash, which profiled as a top-ten CPU
 /// consumer under pandas. Not attacker-relevant: collision resistance
 /// comes from the Python-level hash, exactly as in CPython's own tables.
-pub type DictMap = indexmap::IndexMap<DictKey, Object, crate::fasthash::FxBuildHasher>;
+pub type DictMap = crate::dictmap::DictMap;
 
 /// The global source of [`DictData::mutation_stamp`] values: one
 /// monotonically increasing counter for every dict in the process, so a
@@ -4839,7 +4862,7 @@ impl Extend<(DictKey, Object)> for DictData {
 
 impl IntoIterator for DictData {
     type Item = (DictKey, Object);
-    type IntoIter = indexmap::map::IntoIter<DictKey, Object>;
+    type IntoIter = crate::dictmap::IntoIter;
 
     fn into_iter(self) -> Self::IntoIter {
         self.map.into_iter()
@@ -4848,7 +4871,7 @@ impl IntoIterator for DictData {
 
 impl<'a> IntoIterator for &'a DictData {
     type Item = (&'a DictKey, &'a Object);
-    type IntoIter = indexmap::map::Iter<'a, DictKey, Object>;
+    type IntoIter = crate::dictmap::Iter<'a>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.map.iter()
@@ -4857,7 +4880,7 @@ impl<'a> IntoIterator for &'a DictData {
 
 impl<'a> IntoIterator for &'a mut DictData {
     type Item = (&'a DictKey, &'a mut Object);
-    type IntoIter = indexmap::map::IterMut<'a, DictKey, Object>;
+    type IntoIter = crate::dictmap::IterMut<'a>;
 
     fn into_iter(self) -> Self::IntoIter {
         (**self).iter_mut()
@@ -8969,7 +8992,7 @@ pub enum PyIterator {
         step: Box<BigInt>,
     },
     /// Live iterator over a dict (CPython's `dictiterobject`): walks the
-    /// `IndexMap` by entry index rather than snapshotting, so it pins no
+    /// table by entry index rather than snapshotting, so it pins no
     /// key/value `Rc`s of its own — an overwritten value is freed promptly
     /// even mid-loop (`test_oob_indexing_dictiter_iternextitem` relies on
     /// `d[k] = None` firing the old value's `__del__` during iteration),
diff --git a/crates/weavepy-vm/src/shared_value.rs b/crates/weavepy-vm/src/shared_value.rs
index 64157dc2..189c30f3 100644
--- a/crates/weavepy-vm/src/shared_value.rs
+++ b/crates/weavepy-vm/src/shared_value.rs
@@ -628,6 +628,12 @@ impl SharedStr {
         storage.hash.store(h, Ordering::Relaxed);
         h
     }
+    /// The memoised `hash(s)` if it has been computed.
+    #[inline]
+    pub fn known_hash(this: &Self) -> Option<i64> {
+        let h = this.0.storage().hash.load(Ordering::Relaxed);
+        (h != -1).then_some(h)
+    }
     /// `len(s)` in code points, memoised in the string itself.
     #[inline]
     pub fn char_count(this: &Self) -> usize {
diff --git a/crates/weavepy-vm/src/specialize.rs b/crates/weavepy-vm/src/specialize.rs
index 0a7c9a07..6ab2cee4 100644
--- a/crates/weavepy-vm/src/specialize.rs
+++ b/crates/weavepy-vm/src/specialize.rs
@@ -223,7 +223,7 @@ pub fn attempt_specialize_load_attr(obj: &Object, name: &str) -> InlineCache {
 ///
 /// The fast path takes advantage of two facts:
 ///
-/// 1. The `IndexMap` underneath `DictData` exposes O(1) lookup
+/// 1. The table underneath `DictData` exposes O(1) lookup
 ///    by integer index once we know the slot. So caching the
 ///    slot index lets us skip the hash lookup.
 /// 2. Builtins and globals are stable across dispatches in steady
diff --git a/crates/weavepy-vm/src/stdlib/collections_native.rs b/crates/weavepy-vm/src/stdlib/collections_native.rs
index 729a508e..504e8cbb 100644
--- a/crates/weavepy-vm/src/stdlib/collections_native.rs
+++ b/crates/weavepy-vm/src/stdlib/collections_native.rs
@@ -1420,22 +1420,25 @@ fn tally_one(
     if let Some(probe) = crate::object::LeafProbe::new(key) {
         if !crate::capi_watchers::dicts_active() {
             if let Ok(mut m) = d.try_borrow_mut() {
-                match m.get_mut(&probe) {
-                    Some(Object::Int(n)) if *n < i64::MAX => {
-                        *n += 1;
-                        drop(m);
-                        crate::object::dict_mutation_event(d);
-                        return Ok(());
+                match m.probe_entry(&probe) {
+                    crate::dictmap::ProbeEntry::Occupied(e) => {
+                        if let Object::Int(n) = e.into_mut() {
+                            if *n < i64::MAX {
+                                *n += 1;
+                                drop(m);
+                                crate::object::dict_mutation_event(d);
+                                return Ok(());
+                            }
+                        }
                     }
-                    Some(_) => {}
-                    None if probe.miss_is_exact() => {
-                        m.insert(DictKey(key.clone()), Object::Int(1));
+                    crate::dictmap::ProbeEntry::Vacant(e) if probe.miss_is_exact() => {
+                        e.insert(DictKey(key.clone()), Object::Int(1));
                         drop(m);
                         crate::object::dict_watch_bump(d);
                         crate::object::dict_mutation_event(d);
                         return Ok(());
                     }
-                    None => {}
+                    crate::dictmap::ProbeEntry::Vacant(_) => {}
                 }
             }
         }
diff --git a/crates/weavepy-vm/src/stdlib/collections_odict.rs b/crates/weavepy-vm/src/stdlib/collections_odict.rs
index 5e3ccbae..63f633b6 100644
--- a/crates/weavepy-vm/src/stdlib/collections_odict.rs
+++ b/crates/weavepy-vm/src/stdlib/collections_odict.rs
@@ -6,7 +6,7 @@
 //! CPython's C `odict` keeps a linked list of nodes next to its dict: a
 //! second, keys-only dict in a hidden slot (see [`order_of`]). The order
 //! dict is insertion-ordered like every WeavePy dict, so appending a key,
-//! moving one (`IndexMap::move_index`), and removing one are its own
+//! moving one (`DictMap::move_index`), and removing one are its own
 //! operations. Values are always read from the payload by key, so a
 //! payload changed behind the order's back (`dict.__delitem__(od, k)`)
 //! surfaces as a `KeyError` on the next ordered read, as it does in
diff --git a/crates/weavepy-vm/src/stdlib/elementtree_native.rs b/crates/weavepy-vm/src/stdlib/elementtree_native.rs
index 83e70e27..878ca4f4 100644
--- a/crates/weavepy-vm/src/stdlib/elementtree_native.rs
+++ b/crates/weavepy-vm/src/stdlib/elementtree_native.rs
@@ -1741,14 +1741,12 @@ struct PathKey<'a> {
     exotic: std::cell::Cell<bool>,
 }
 
-impl std::hash::Hash for PathKey<'_> {
-    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-        self.hash.hash(state);
+impl crate::dictmap::Probe for PathKey<'_> {
+    fn probe_hash(&self) -> i64 {
+        self.hash
     }
-}
 
-impl indexmap::Equivalent<DictKey> for PathKey<'_> {
-    fn equivalent(&self, key: &DictKey) -> bool {
+    fn probe_eq(&self, key: &DictKey) -> bool {
         match &key.0 {
             Object::Tuple(t) => {
                 let mut items = t.iter();
diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs
index 85724186..ba7777c9 100644
--- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs
+++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs
@@ -14,10 +14,8 @@
 mod classes;
 mod encode;
 
-use std::hash::BuildHasher;
 
 use crate::shared_value::{SharedSlice, SharedStr};
-use indexmap::map::raw_entry_v1::{RawEntryApiV1, RawEntryMut};
 use num_bigint::BigInt;
 use weavepy_compiler::CodeObject;
 
@@ -1149,15 +1147,15 @@ impl<'c> Decoder<'c> {
             };
             // `insert` for a string key, which equals only another string:
             // the cached hash and a direct comparison, in line.
-            let hash = crate::fasthash::FxBuildHasher.hash_one(SharedStr::hash_cached(&name));
-            match map.raw_entry_mut_v1().from_hash(hash, |key| {
+            let hash = SharedStr::hash_cached(&name);
+            match map.find_by_hash(hash, |key| {
                 matches!(&key.0, Object::Str(other) if SharedStr::ptr_eq(other, &name) || *other == name)
             }) {
-                RawEntryMut::Occupied(mut entry) => {
-                    entry.insert(value);
+                Some(i) => {
+                    *map.get_index_mut(i).expect("found above").1 = value;
                 }
-                RawEntryMut::Vacant(entry) => {
-                    entry.insert_hashed_nocheck(hash, DictKey(Object::Str(name)), value);
+                None => {
+                    map.insert_unique_hashed(hash, DictKey(Object::Str(name)), value);
                 }
             }
         }
@@ -1273,7 +1271,7 @@ impl<'c> Decoder<'c> {
             let table: &mut DictMap = &mut state;
             // An interned key is equal to the key it replaces and has the
             // same hash, so the table stays valid.
-            for (key, _) in indexmap::map::MutableKeys::iter_mut2(&mut *table) {
+            for (key, _) in table.iter_mut2() {
                 let Object::Str(name) = &key.0 else {
                     return None;
                 };
diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel/classes.rs b/crates/weavepy-vm/src/stdlib/pickle_accel/classes.rs
index f2b7ab3c..89784542 100644
--- a/crates/weavepy-vm/src/stdlib/pickle_accel/classes.rs
+++ b/crates/weavepy-vm/src/stdlib/pickle_accel/classes.rs
@@ -5,7 +5,6 @@
 //! could require a Python callback reports `None` so that the caller returns
 //! to the full pickler or unpickler.
 
-use std::hash::BuildHasher;
 
 use crate::builtin_types::builtin_types;
 use crate::object::{py_str_hash, DictData, Object, StrKey};
@@ -65,12 +64,10 @@ pub(super) fn ambient_state_is_plain() -> bool {
 /// The outer `None` reports a stored key that isn't an exact `str` in the
 /// probe sequence. Comparing with it could call a Python `__eq__`.
 pub(super) fn pure_get(dict: &DictData, name: &str) -> Option<Option<Object>> {
-    use indexmap::map::raw_entry_v1::RawEntryApiV1;
-    let hash = crate::fasthash::FxBuildHasher.hash_one(py_str_hash(name));
+    let hash = py_str_hash(name);
     let mut exotic = false;
     let found = dict
-        .raw_entry_v1()
-        .from_hash(hash, |key| match &key.0 {
+        .find_by_hash(hash, |key| match &key.0 {
             Object::Str(stored) => stored.as_ref() == name,
             Object::WStr(_) => false,
             _ => {
@@ -78,6 +75,7 @@ pub(super) fn pure_get(dict: &DictData, name: &str) -> Option<Option<Object>> {
                 false
             }
         })
+        .and_then(|i| dict.get_index(i))
         .map(|(_, value)| value.clone());
     if exotic {
         None
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index 741c5c77..33359819 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -11218,9 +11218,6 @@ unsafe extern "C" fn wpjit_attr_set(frame: *mut JitFrame, pin: i64, site: i64) -
         // execution. Watched instance dicts deopt so the generic path
         // fires the exact watcher events.
         AttrStorage::NewKey => {
-            use indexmap::map::raw_entry_v1::{RawEntryApiV1, RawEntryMut};
-            use std::hash::BuildHasher;
-
             if crate::capi_watchers::dicts_active() {
                 return 1;
             }
@@ -11249,25 +11246,22 @@ unsafe extern "C" fn wpjit_attr_set(frame: *mut JitFrame, pin: i64, site: i64) -
                 &mut **dict
             };
             let probe = crate::object::LeafNameProbe::new(&g.name, g.name_hash);
-            let hash = crate::fasthash::FxBuildHasher.hash_one(g.name_hash);
-            let entry = dict
-                .raw_entry_mut_v1()
-                .from_key_hashed_nocheck(hash, &probe);
+            let entry = dict.probe_entry(&probe);
             // A hash-colliding user key could run Python during equality.
             // Resume before the store so the interpreter owns that callback.
             if probe.saw_exotic() {
                 return 1;
             }
             let old = match entry {
-                RawEntryMut::Occupied(mut entry) => {
+                crate::dictmap::ProbeEntry::Occupied(mut entry) => {
                     let dst = entry.get_mut();
                     // The displaced-value discipline of the indexed arm.
                     Some(std::mem::replace(dst, v))
                 }
-                RawEntryMut::Vacant(entry) => {
+                crate::dictmap::ProbeEntry::Vacant(entry) => {
                     // The guard already owns the interned key. Reuse the
-                    // probe's hash and vacant entry instead of probing again.
-                    entry.insert_hashed_nocheck(hash, DictKey(Object::Str(g.name.clone())), v);
+                    // probe's hash and vacant slot instead of probing again.
+                    entry.insert(DictKey(Object::Str(g.name.clone())), v);
                     None
                 }
             };
diff --git a/tests/regrtest/test_dict_table.py b/tests/regrtest/test_dict_table.py
new file mode 100644
index 00000000..ea8dc89b
--- /dev/null
+++ b/tests/regrtest/test_dict_table.py
@@ -0,0 +1,187 @@
+"""The ordered hash table behind `dict`: order, growth, removal, and keys.
+
+Covers the shapes the table handles differently: small tables (no slot
+index), the switch to an indexed table and its growth, removal from the
+front, middle and end (the later entries renumber), keys whose hashes
+collide (a key's `__eq__` runs only against keys of equal hash, as in
+CPython), a `__eq__` that mutates the dict mid-probe, and the positional
+users of the table (iteration, `popitem`, `OrderedDict.move_to_end`, the
+`del`-then-reinsert iteration trip-wire, `copy`, and pickling order).
+"""
+
+import pickle
+import unittest
+from collections import OrderedDict
+
+
+class Collide:
+    """A key with a fixed hash that counts its `__eq__` calls."""
+
+    calls = 0
+
+    def __init__(self, name, h=42):
+        self.name = name
+        self.h = h
+
+    def __hash__(self):
+        return self.h
+
+    def __eq__(self, other):
+        Collide.calls += 1
+        return isinstance(other, Collide) and self.name == other.name
+
+    def __repr__(self):
+        return f"Collide({self.name!r})"
+
+
+class TableTest(unittest.TestCase):
+    def test_growth_keeps_order_and_lookups(self):
+        d = {}
+        for i in range(2000):
+            d[i * 1024] = i
+            if i in (7, 8, 9, 100, 1000):
+                self.assertEqual(list(d), [k * 1024 for k in range(i + 1)])
+        for i in range(2000):
+            self.assertEqual(d[i * 1024], i)
+        self.assertNotIn(5, d)
+        self.assertEqual(len(d), 2000)
+
+    def test_removal_renumbers(self):
+        for n in (5, 9, 40, 500):
+            d = {f"k{i}": i for i in range(n)}
+            keys = list(d)
+            # Front, middle and end.
+            for k in (keys[0], keys[n // 2], keys[-1]):
+                del d[k]
+                keys.remove(k)
+                self.assertEqual(list(d), keys)
+                for kk in keys:
+                    self.assertEqual(d[kk], int(kk[1:]))
+            # Every other one.
+            for k in keys[::2]:
+                self.assertEqual(d.pop(k), int(k[1:]))
+            keys = keys[1::2]
+            self.assertEqual(list(d), keys)
+            d["new"] = -1
+            self.assertEqual(list(d)[-1], "new")
+            self.assertEqual(d.popitem(), ("new", -1))
+
+    def test_reinsert_goes_last(self):
+        d = dict.fromkeys(range(20))
+        del d[3]
+        d[3] = "x"
+        self.assertEqual(list(d)[-1], 3)
+        self.assertEqual(list(d)[:3], [0, 1, 2])
+
+    def test_collisions_compare_only_equal_hashes(self):
+        a, b, c = Collide("a"), Collide("b"), Collide("c", h=7)
+        d = {a: 1, c: 3}
+        Collide.calls = 0
+        d[b] = 2
+        # `b` meets `a` (same hash) once; `c` (other hash) never.
+        self.assertEqual(Collide.calls, 1)
+        Collide.calls = 0
+        self.assertEqual(d[Collide("c", h=7)], 3)
+        self.assertEqual(Collide.calls, 1)
+        self.assertEqual(list(d), [a, c, b])
+        many = {Collide(str(i)): i for i in range(50)}
+        for i in range(50):
+            self.assertEqual(many[Collide(str(i))], i)
+        del many[Collide("10")]
+        self.assertNotIn(Collide("10"), many)
+        self.assertEqual(len(many), 49)
+
+    def test_eq_mutating_dict(self):
+        class Evil:
+            def __init__(self, d):
+                self.d = d
+
+            def __hash__(self):
+                return 1
+
+            def __eq__(self, other):
+                self.d.clear()
+                return False
+
+        d = {}
+        d[Evil(d)] = 1
+        # The probe meets the stored key (equal hash), whose `__eq__`
+        # empties the dict; CPython restarts and finds nothing.
+        self.assertNotIn(Evil(d), d)
+        self.assertEqual(len(d), 0)
+
+    def test_unhashable_and_mixed_numeric_keys(self):
+        d = {1: "int", 2.5: "float", "s": "str", (1, 2): "tuple"}
+        self.assertEqual(d[1.0], "int")
+        self.assertEqual(d[True], "int")
+        self.assertEqual(d[(1, 2)], "tuple")
+        with self.assertRaises(TypeError):
+            d[[1]] = 1
+        nan = float("nan")
+        d[nan] = "nan"
+        self.assertEqual(d[nan], "nan")
+
+    def test_iteration_tripwires(self):
+        d = {i: i for i in range(12)}
+        with self.assertRaises(RuntimeError):
+            for k in d:
+                d[k + 100] = 0
+        d = {i: i for i in range(12)}
+        with self.assertRaises(RuntimeError):
+            for k in d:
+                del d[k]
+                d[k + 100] = 0
+
+    def test_copy_update_clear(self):
+        src = {f"k{i}": i for i in range(30)}
+        c = src.copy()
+        c["k3"] = "changed"
+        self.assertEqual(src["k3"], 3)
+        self.assertEqual(list(c), list(src))
+        c.clear()
+        self.assertEqual(c, {})
+        c.update(src)
+        self.assertEqual(c, src)
+        c["extra"] = 1
+        self.assertEqual(list(c)[-1], "extra")
+
+    def test_ordered_dict_moves(self):
+        od = OrderedDict((i, str(i)) for i in range(30))
+        od.move_to_end(5)
+        od.move_to_end(20, last=False)
+        keys = list(od)
+        self.assertEqual(keys[-1], 5)
+        self.assertEqual(keys[0], 20)
+        self.assertEqual(od.popitem(last=False), (20, "20"))
+        self.assertEqual(od.popitem(), (5, "5"))
+        for k in range(30):
+            if k not in (5, 20):
+                self.assertEqual(od[k], str(k))
+        lru = OrderedDict()
+        for i in range(500):
+            k = (i * 7) % 37
+            lru[k] = i
+            lru.move_to_end(k)
+            if len(lru) > 20:
+                lru.popitem(last=False)
+        self.assertEqual(len(lru), 20)
+        self.assertEqual(list(lru)[-1], (499 * 7) % 37)
+
+    def test_pickle_order(self):
+        d = {f"k{i}": i for i in range(40)}
+        for k in list(d)[::3]:
+            del d[k]
+        r = pickle.loads(pickle.dumps(d))
+        self.assertEqual(list(r.items()), list(d.items()))
+
+    def test_setdefault_and_get(self):
+        d = {}
+        for i in range(100):
+            self.assertEqual(d.setdefault(i % 10, i), i % 10)
+        self.assertEqual(len(d), 10)
+        self.assertEqual(d.get(99), None)
+        self.assertEqual(d.get(99, "x"), "x")
+
+
+if __name__ == "__main__":
+    unittest.main()

From fcdb5f6da9a82b27b065c3ff06292b5dd04b80a1 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 01:29:20 -0700
Subject: [PATCH 033/173] perf: settle str and int dict probes without scopes
 or rehashing

- dict_remove (del d[k], dict.pop, namedtuple._replace's pops) takes a
  str or int key through one native probe, without the two comparison
  scopes (thread-local state) the general path enters.
- functools.lru_cache's scalar path probes with a LeafProbe instead of
  a cloned DictKey under key_cmp_deferred, and a proven miss inserts on
  the probe's hash without probing again.
- The subscript fast paths (frame JIT, core loop, dict.get kernel) probe
  in line (DictMap::get_hot), and probe_entry is always in line.
- An MRO walk by name (TypeObject::lookup_cached) hashes the name at most
  once for the whole walk (StrKeyLazy) instead of once per class dict.
- With the hash seed pinned, the SipHash of a byte string of up to 32
  bytes is memoized per thread by content (a direct-mapped table), so a
  &str probe of a large dict, or a fresh string's first hash, pays a
  short compare on a repeat.

Instructions per call, pinned seed, against the previous commit: suite
geomean 0.995 (1.560x -> 1.551x CPython); closures_decorators 0.973,
collections_mix 0.980, deepcopy 0.984, sudoku 0.986, logging_mix 0.988.
---
 crates/weavepy-vm/src/builtins.rs             |  20 ++++
 crates/weavepy-vm/src/dictmap.rs              |  19 +++
 crates/weavepy-vm/src/frame_jit.rs            |   4 +-
 crates/weavepy-vm/src/lib.rs                  |   2 +-
 crates/weavepy-vm/src/object.rs               | 110 ++++++++++++++++++
 crates/weavepy-vm/src/stdlib/functools_mod.rs |  44 +++++--
 crates/weavepy-vm/src/types.rs                |   3 +-
 7 files changed, 189 insertions(+), 13 deletions(-)

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 8526cd18..e1cf5d0b 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -13611,6 +13611,26 @@ pub(crate) fn dict_remove(
     d: &Rc<RefCell<DictData>>,
     key: &Object,
 ) -> Result<Option<(Object, Object)>, RuntimeError> {
+    // A `str` or `int` key settles by native equality: one probe, with no
+    // comparison scope (see `dict_lookup`).
+    if let Some(probe) = crate::object::LeafProbe::new(key) {
+        if let Ok(mut m) = d.try_borrow_mut() {
+            match m.get_index_of(&probe) {
+                Some(i) => {
+                    let (k, v) = m.shift_remove_index(i).expect("found above");
+                    drop(m);
+                    crate::object::dict_watch_bump(d);
+                    crate::object::dict_mutation_event(d);
+                    if crate::capi_watchers::dicts_active() {
+                        crate::capi_watchers::dict_event("DELETED", d, Some(key), None);
+                    }
+                    return Ok(Some((k.0, v)));
+                }
+                None if probe.miss_is_exact() => return Ok(None),
+                None => {}
+            }
+        }
+    }
     if crate::object::dict_key_is_reentrant(key) {
         return crate::object::dict_reentrant_remove(d, key);
     }
diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index ae6ddf71..2cfc7432 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -434,6 +434,24 @@ impl DictMap {
         Some(found)
     }
 
+    /// [`Self::get`], always in line: for the interpreter's subscript
+    /// fast paths, where the call would cost a fair share of the probe.
+    #[inline(always)]
+    pub fn get_hot<Q: Probe + ?Sized>(&self, key: &Q) -> Option<&Object> {
+        if self.slots.is_empty() {
+            if self.entries.is_empty() {
+                return None;
+            }
+            if let Some(found) = self.scan_unhashed(key) {
+                // SAFETY: a found position is in bounds.
+                return found.map(|i| unsafe { &self.entries.get_unchecked(i).value });
+            }
+        }
+        let i = self.find_hashed(key.probe_hash(), key).ok()?;
+        // SAFETY: a found position is in bounds.
+        Some(unsafe { &self.entries.get_unchecked(i).value })
+    }
+
     #[inline]
     pub fn get<Q: Probe + ?Sized>(&self, key: &Q) -> Option<&Object> {
         let i = self.get_index_of(key)?;
@@ -628,6 +646,7 @@ impl DictMap {
 
     /// The entry for a probe: its position if present, else a vacancy
     /// that inserts the key the caller builds.
+    #[inline(always)]
     pub fn probe_entry<Q: Probe + ?Sized>(&mut self, key: &Q) -> ProbeEntry<'_> {
         let hash = key.probe_hash();
         match self.find_hashed(hash, key) {
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index f88c1769..fea39257 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1681,7 +1681,7 @@ unsafe extern "C" fn k_dict_get(
         let Some(m) = d.peek() else {
             return 1;
         };
-        let v = match m.get(&probe) {
+        let v = match m.get_hot(&probe) {
             Some(v) => crate::clone_hot(v),
             None if probe.miss_is_exact() => {
                 if b.is_null() {
@@ -2227,7 +2227,7 @@ unsafe fn subscr_fast(
                 let Some(probe) = crate::object::LeafProbe::new(k) else {
                     return 1;
                 };
-                let Some(v) = d.peek().and_then(|m| m.get(&probe)) else {
+                let Some(v) = d.peek().and_then(|m| m.get_hot(&probe)) else {
                     return 1;
                 };
                 crate::clone_hot(v)
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ca50babe..d3051d5d 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -17498,7 +17498,7 @@ impl Interpreter {
                         // SAFETY: a read between two instructions (the
                         // probe runs no code).
                         let d = unsafe { d.peek() }?;
-                        clone_hot(d.get(&probe)?)
+                        clone_hot(d.get_hot(&probe)?)
                     }
                     // A string's code point (`str_char_at` byte-indexes an
                     // ASCII string and walks any other from its cursor).
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 97d6b4e3..5080f27d 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -3348,6 +3348,48 @@ impl crate::dictmap::Probe for StrKey<'_> {
     }
 }
 
+/// [`StrKey`] for a probe of several tables (an MRO walk): the name's hash
+/// is computed on first need and kept, so the walk hashes it at most once,
+/// and not at all while it meets only small tables (which compare names
+/// directly; see [`crate::dictmap::Probe::probe_eq_unhashed`]).
+#[derive(Debug)]
+pub struct StrKeyLazy<'a> {
+    pub s: &'a str,
+    hash: std::cell::Cell<Option<i64>>,
+}
+
+impl<'a> StrKeyLazy<'a> {
+    #[inline]
+    pub fn new(s: &'a str) -> Self {
+        Self {
+            s,
+            hash: std::cell::Cell::new(None),
+        }
+    }
+}
+
+impl crate::dictmap::Probe for StrKeyLazy<'_> {
+    #[inline]
+    fn probe_hash(&self) -> i64 {
+        if let Some(h) = self.hash.get() {
+            return h;
+        }
+        let h = py_str_hash(self.s);
+        self.hash.set(Some(h));
+        h
+    }
+
+    #[inline]
+    fn probe_eq(&self, key: &DictKey) -> bool {
+        crate::dictmap::Probe::probe_eq(&StrKey(self.s), key)
+    }
+
+    #[inline]
+    fn probe_eq_unhashed(&self, key: &DictKey) -> Option<bool> {
+        crate::dictmap::Probe::probe_eq_unhashed(&StrKey(self.s), key)
+    }
+}
+
 /// [`StrKey`] with its Python hash precomputed (RFC 0077 WS4). The
 /// interpreter's attribute-name probes hash the same `co_names` entry on
 /// every execution of a site — a siphash per probe — so the VM memoises
@@ -12817,11 +12859,79 @@ fn siphash13(k0: u64, k1: u64, data: &[u8]) -> u64 {
     (v0 ^ v1) ^ (v2 ^ v3)
 }
 
+/// The longest byte string [`sip_memo`] remembers.
+const SIP_MEMO_MAX: usize = 32;
+
+/// One remembered SipHash result: the bytes (`len` of them; `0` marks an
+/// empty slot) and their hash.
+#[derive(Clone, Copy)]
+struct SipMemoEntry {
+    len: u8,
+    hash: i64,
+    bytes: [u8; SIP_MEMO_MAX],
+}
+
+const SIP_MEMO_SLOTS: usize = 512;
+
+thread_local! {
+    /// A direct-mapped memo of the SipHash of short byte strings, by
+    /// content. With the seed pinned, a name hashed from a `&str` (a
+    /// probe of a large dict by `StrKey`, a fresh string's first hash)
+    /// costs a short compare instead of a SipHash run on a repeat; CPython
+    /// gets the same effect from the hash cached in each interned name.
+    static SIP_MEMO: std::cell::UnsafeCell<[SipMemoEntry; SIP_MEMO_SLOTS]> = const {
+        std::cell::UnsafeCell::new([SipMemoEntry { len: 0, hash: 0, bytes: [0; SIP_MEMO_MAX] }; SIP_MEMO_SLOTS])
+    };
+}
+
+/// `compute(bytes)`, remembered per thread for `bytes` of 1 to
+/// [`SIP_MEMO_MAX`] bytes.
+#[inline]
+fn sip_memo(bytes: &[u8], compute: impl FnOnce(&[u8]) -> i64) -> i64 {
+    let n = bytes.len();
+    debug_assert!((1..=SIP_MEMO_MAX).contains(&n));
+    // SAFETY: the loads stay inside `bytes` (overlapping for short ones).
+    let fold = unsafe {
+        let p = bytes.as_ptr();
+        if n >= 8 {
+            p.cast::<u64>().read_unaligned()
+                ^ p.add(n - 8).cast::<u64>().read_unaligned().rotate_left(29)
+        } else if n >= 4 {
+            u64::from(p.cast::<u32>().read_unaligned())
+                | u64::from(p.add(n - 4).cast::<u32>().read_unaligned()) << 32
+        } else {
+            u64::from(*p) | u64::from(*p.add(n / 2)) << 8 | u64::from(*p.add(n - 1)) << 16
+        }
+    };
+    let slot = ((fold ^ n as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 55) as usize;
+    // SAFETY (both accesses): this thread's memo, borrowed only for the
+    // statement (nothing inside runs code that could reach it).
+    let hit = SIP_MEMO.try_with(|memo| {
+        let e = unsafe { &(*memo.get())[slot] };
+        (usize::from(e.len) == n && crate::dictmap::key_bytes_eq(bytes, &e.bytes[..n]))
+            .then_some(e.hash)
+    });
+    if let Ok(Some(h)) = hit {
+        return h;
+    }
+    let h = compute(bytes);
+    let _ = SIP_MEMO.try_with(|memo| {
+        let e = unsafe { &mut (*memo.get())[slot] };
+        e.len = n as u8;
+        e.hash = h;
+        e.bytes[..n].copy_from_slice(bytes);
+    });
+    h
+}
+
 fn py_hash_bytes_slice(bytes: &[u8]) -> i64 {
     if bytes.is_empty() {
         return 0;
     }
     let v = match hash_algo() {
+        HashAlgo::Sip { k0, k1 } if bytes.len() <= SIP_MEMO_MAX => {
+            sip_memo(bytes, |b| siphash13(*k0, *k1, b) as i64)
+        }
         HashAlgo::Sip { k0, k1 } => siphash13(*k0, *k1, bytes) as i64,
         HashAlgo::Fx { salt } => {
             use std::hash::Hasher;
diff --git a/crates/weavepy-vm/src/stdlib/functools_mod.rs b/crates/weavepy-vm/src/stdlib/functools_mod.rs
index 219f5684..f393e5d3 100644
--- a/crates/weavepy-vm/src/stdlib/functools_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/functools_mod.rs
@@ -863,14 +863,32 @@ fn scalar_lru_operation(
             note_scalar_lru_event(2);
             return Ok(ScalarLru::Slow);
         }
-        // Deferral also protects against private-cache tampering: a foreign
-        // key must not invoke Python inside the paired borrow.
-        let (found, deferred) =
-            crate::object::key_cmp_deferred(|| data.get_index_of(&DictKey(key.clone())));
-        let found = found?;
-        if deferred {
-            return Err(invalid_lru_order());
-        }
+        // A `str` or `int` key settles by native equality in one probe;
+        // its hash is kept for the insert below.
+        let probe = crate::object::LeafProbe::new(key);
+        let exact = match &probe {
+            Some(p) => match data.get_index_of(p) {
+                Some(i) => Some(Some(i)),
+                None if p.miss_is_exact() => Some(None),
+                None => None,
+            },
+            None => None,
+        };
+        let (found, exact_miss) = match exact {
+            Some(found) => (found, found.is_none()),
+            None => {
+                // Deferral also protects against private-cache tampering: a
+                // foreign key must not invoke Python inside the paired borrow.
+                let (found, deferred) = crate::object::key_cmp_deferred(|| {
+                    data.get_index_of(&DictKey(key.clone()))
+                });
+                let found = found?;
+                if deferred {
+                    return Err(invalid_lru_order());
+                }
+                (found, false)
+            }
+        };
         if let Some(index) = found {
             let (stored, value) = data.get_index(index).ok_or_else(invalid_lru_order)?;
             if !native_lru_key(&stored.0) {
@@ -903,7 +921,15 @@ fn scalar_lru_operation(
         } else {
             None
         };
-        data.insert(DictKey(key.clone()), value.clone());
+        match &probe {
+            // Proven absent above (an eviction keeps it so): no second probe.
+            Some(p) if exact_miss => {
+                data.insert_unique_hashed(p.hash, DictKey(key.clone()), value.clone());
+            }
+            _ => {
+                data.insert(DictKey(key.clone()), value.clone());
+            }
+        }
         order.push(limit).ok_or_else(invalid_lru_order)?;
     }
     // Publish a consistent cache before releasing an evicted result.
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 72cf11ae..8f48a7cf 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -1927,7 +1927,8 @@ impl TypeObject {
             }
             None => {}
         }
-        let key = crate::object::StrKey(name);
+        // (Hashed at most once for the whole walk.)
+        let key = crate::object::StrKeyLazy::new(name);
         let mro = self.mro.borrow();
         for (i, owner) in mro.iter().enumerate() {
             let d = owner.dict.borrow();

From e4445007c9d8bb1973199aa984ce0a9f037d6418 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 02:21:36 -0700
Subject: [PATCH 034/173] perf: set OrderedDict items with one probe per table

od[key] = value's leaf half probes the payload and the order dict once
each through probe_entry and fills a miss from the probe's vacancy,
instead of probing both, then probing again to insert. It still settles
both probes before changing either table.

Also fixes the dictmap unit tests' asserts (Object has no PartialEq).

Instructions per call, pinned seed: od[k] = v; move_to_end loop 1787 ->
1732 per iteration, the LRU pattern 7990 -> 7739; collections_mix
0.992; suite geomean neutral (1.0006, quick-build noise).
---
 crates/weavepy-vm/src/dictmap.rs              |  4 +--
 .../src/stdlib/collections_odict.rs           | 29 ++++++++++---------
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index 2cfc7432..e55f611b 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -1252,7 +1252,7 @@ mod tests {
     fn insert_remove_order() {
         let mut m = DictMap::new();
         for i in 0..200 {
-            assert_eq!(m.insert(k(i * 1024), Object::Int(i)), None);
+            assert!(m.insert(k(i * 1024), Object::Int(i)).is_none());
             check(&m);
         }
         for i in (0..200).step_by(3) {
@@ -1292,6 +1292,6 @@ mod tests {
         let c = m.clone();
         check(&c);
         assert_eq!(c.get(&k(5)).map(|o| matches!(o, Object::Int(5))), Some(true));
-        assert_eq!(c.get(&k(50)).is_none(), true);
+        assert!(c.get(&k(50)).is_none());
     }
 }
diff --git a/crates/weavepy-vm/src/stdlib/collections_odict.rs b/crates/weavepy-vm/src/stdlib/collections_odict.rs
index 63f633b6..984de777 100644
--- a/crates/weavepy-vm/src/stdlib/collections_odict.rs
+++ b/crates/weavepy-vm/src/stdlib/collections_odict.rs
@@ -249,33 +249,36 @@ fn od_setitem_leaf(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
     let probe = crate::object::LeafProbe::new(key)?;
     let mut m = main.try_borrow_mut().ok()?;
     let mut o = order.try_borrow_mut().ok()?;
-    let in_main = m.get_index_of(&probe);
-    let in_order = o.get_index_of(&probe).is_some();
-    if (in_main.is_none() || !in_order) && !probe.miss_is_exact() {
+    // One probe of each finds the entry or the slot a new key takes.
+    use crate::dictmap::ProbeEntry;
+    let in_main = m.probe_entry(&probe);
+    let in_order = o.probe_entry(&probe);
+    let (main_new, order_new) = (
+        matches!(in_main, ProbeEntry::Vacant(_)),
+        matches!(in_order, ProbeEntry::Vacant(_)),
+    );
+    if (main_new || order_new) && !probe.miss_is_exact() {
         return None;
     }
     let old = match in_main {
-        Some(i) => {
-            let (_, slot) = m.get_index_mut(i)?;
-            Some(std::mem::replace(slot, value.clone()))
-        }
-        None => {
-            m.insert(DictKey(key.clone()), value.clone());
+        ProbeEntry::Occupied(e) => Some(std::mem::replace(e.into_mut(), value.clone())),
+        ProbeEntry::Vacant(e) => {
+            e.insert(DictKey(key.clone()), value.clone());
             None
         }
     };
-    if !in_order {
-        o.insert(DictKey(key.clone()), Object::None);
+    if let ProbeEntry::Vacant(e) = in_order {
+        e.insert(DictKey(key.clone()), Object::None);
     }
     drop((m, o));
     let changed = old.as_ref().is_none_or(|o| !o.is_same(value));
-    if in_main.is_none() {
+    if main_new {
         crate::object::dict_watch_bump(main);
     }
     if changed {
         crate::object::dict_mutation_event(main);
     }
-    if !in_order {
+    if order_new {
         crate::object::dict_watch_bump(&order);
     }
     drop(old);

From 467c3047e30ff16a17c01e4572d6cafcbb13eb63 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 03:43:00 -0700
Subject: [PATCH 035/173] perf: test an in-line release's gates with one load

Compiled code released a shared value in line only after testing four
flags: the reference-count bias's shared flag, the dict and function
watcher flags, and the reftrace flag. Each test was a 64-bit address
constant, a byte load, a compare and an AND at every release site.

The flags now sit side by side in one aligned `ReleaseGates` static, so
a single 32-bit load tests them all. The modules that own them keep
their names for the bytes. The frame JIT's statistics also report each
compiled function's code size.
---
 crates/weavepy-vm/src/capi_watchers.rs        | 14 ++----------
 crates/weavepy-vm/src/frame_jit.rs            | 20 ++++++++++-------
 crates/weavepy-vm/src/lib.rs                  | 16 +++++---------
 .../src/stdlib/testinternalcapi_mod.rs        |  8 +------
 crates/weavepy-vm/src/sync.rs                 | 22 +++++++++++++++++--
 5 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/crates/weavepy-vm/src/capi_watchers.rs b/crates/weavepy-vm/src/capi_watchers.rs
index a3a343ca..0e1b6d7e 100644
--- a/crates/weavepy-vm/src/capi_watchers.rs
+++ b/crates/weavepy-vm/src/capi_watchers.rs
@@ -21,11 +21,11 @@ use parking_lot::Mutex;
 use std::sync::atomic::{AtomicBool, Ordering};
 
 /// Any dict currently watched?
-static DICTS_ACTIVE: AtomicBool = AtomicBool::new(false);
+static DICTS_ACTIVE: &AtomicBool = &crate::sync::RELEASE_GATES.dicts_watched;
 /// Any type currently watched?
 static TYPES_ACTIVE: AtomicBool = AtomicBool::new(false);
 /// Any function watcher registered?
-static FUNCS_ACTIVE: AtomicBool = AtomicBool::new(false);
+static FUNCS_ACTIVE: &AtomicBool = &crate::sync::RELEASE_GATES.funcs_watched;
 
 /// A watched dict: identity pointer, the bitmask of watcher IDs watching
 /// it, and a keep-check `Weak` so a stale entry (the dict died on a path
@@ -100,16 +100,6 @@ fn call_dispatcher(cb: &Object, args: Vec<Object>) {
 
 // ---------------------------------------------------------------- dicts
 
-#[inline(always)]
-/// The dict and function watcher flags' bytes (nonzero while any is
-/// registered), for compiled code that tests them in line.
-pub(crate) fn release_flags() -> [*const u8; 2] {
-    [
-        DICTS_ACTIVE.as_ptr().cast_const().cast(),
-        FUNCS_ACTIVE.as_ptr().cast_const().cast(),
-    ]
-}
-
 pub fn dicts_active() -> bool {
     DICTS_ACTIVE.load(Ordering::Relaxed)
 }
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index fea39257..4dfe687f 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -5532,8 +5532,12 @@ fn compile_with(
     let blocks = engine.ctx.func.dfg.num_blocks();
     let defined = engine.module.define_function(id, &mut engine.ctx);
     if let (Some(t0), Some(t1)) = (t0, t1) {
+        let bytes = engine
+            .ctx
+            .compiled_code()
+            .map_or(0, |c| c.code_buffer().len());
         eprintln!(
-            "frame jit: {} lowered in {:?} ({insts} IR instructions, {blocks} blocks{}), compiled in {:?}",
+            "frame jit: {} lowered in {:?} ({insts} IR instructions, {blocks} blocks{}, {bytes} bytes), compiled in {:?}",
             code.qualname,
             t1 - t0,
             if lean { ", lean" } else { "" },
@@ -7467,13 +7471,13 @@ impl<'a> Lower<'a> {
             .band_imm(bit, self.tags.rc_counted | self.tags.rc_counted_hdr);
         let rc = self.b.ins().icmp_imm(IntCC::NotEqual, rc, 0);
         // The single-thread bias, and the release observers, all off.
-        let mut quiet = rc;
-        for flag in crate::plain_release_flags() {
-            let addr = self.b.ins().iconst(self.ptr, flag as i64);
-            let on = self.b.ins().load(types::I8, FLAGS, addr, 0);
-            let off = self.b.ins().icmp_imm(IntCC::Equal, on, 0);
-            quiet = self.b.ins().band(quiet, off);
-        }
+        let addr = self
+            .b
+            .ins()
+            .iconst(self.ptr, crate::plain_release_gate() as i64);
+        let on = self.b.ins().load(types::I32, FLAGS, addr, 0);
+        let off = self.b.ins().icmp_imm(IntCC::Equal, on, 0);
+        let quiet = self.b.ins().band(rc, off);
         self.b.ins().brif(quiet, counted, &[], slow, &[]);
         self.b.switch_to_block(counted);
         let w = self.count_addr(local, bit);
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index d3051d5d..20ecac65 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -71940,18 +71940,12 @@ pub(crate) fn native_site(ext: &CodeConstObjects, pc: usize) -> bool {
         .is_some_and(|s| s.load(std::sync::atomic::Ordering::Relaxed))
 }
 
-/// The bytes compiled code tests before releasing a shared value in line:
+/// The word compiled code tests before releasing a shared value in line:
 /// the reference-count bias's shared flag and every observer
-/// [`Interpreter::core_drops_plain`] consults; all zero for a plain
-/// decrement.
-pub(crate) fn plain_release_flags() -> [*const u8; 4] {
-    let [dicts, funcs] = crate::capi_watchers::release_flags();
-    [
-        crate::sync::rc_shared_flag().as_ptr().cast_const().cast(),
-        dicts,
-        funcs,
-        crate::stdlib::testinternalcapi_mod::reftrace_print_flag(),
-    ]
+/// [`Interpreter::core_drops_plain`] consults, one byte each; zero for a
+/// plain decrement.
+pub(crate) fn plain_release_gate() -> *const u32 {
+    std::ptr::from_ref(&crate::sync::RELEASE_GATES).cast()
 }
 
 /// `v.clone()`, in line (see `Object`'s `Clone`).
diff --git a/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs
index 89cd0e90..a58ebd45 100644
--- a/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs
@@ -313,13 +313,7 @@ fn remove_mem_hooks(_args: &[Object]) -> Result<Object, RuntimeError> {
 // reap chokepoints.
 // ------------------------------------------------------------------
 
-static REFTRACE_PRINT: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
-
-/// [`reftrace_print_active`]'s byte (nonzero while active), for compiled
-/// code that tests it in line.
-pub(crate) fn reftrace_print_flag() -> *const u8 {
-    REFTRACE_PRINT.as_ptr().cast_const().cast()
-}
+static REFTRACE_PRINT: &std::sync::atomic::AtomicBool = &crate::sync::RELEASE_GATES.reftrace_print;
 
 /// Consulted by the VM's allocation/reap chokepoints.
 #[inline]
diff --git a/crates/weavepy-vm/src/sync.rs b/crates/weavepy-vm/src/sync.rs
index 663a68b5..e45553fd 100644
--- a/crates/weavepy-vm/src/sync.rs
+++ b/crates/weavepy-vm/src/sync.rs
@@ -416,15 +416,33 @@ pub(crate) fn cells_unguarded_flag() -> *const bool {
     CELLS_UNGUARDED.as_ptr()
 }
 
+/// The flags a plain in-line release must find all clear (see
+/// [`crate::plain_release_gate`]), side by side so that compiled code
+/// tests them with one 32-bit load.
+#[repr(C, align(4))]
+pub(crate) struct ReleaseGates {
+    pub(crate) rc_shared: std::sync::atomic::AtomicBool,
+    pub(crate) dicts_watched: std::sync::atomic::AtomicBool,
+    pub(crate) funcs_watched: std::sync::atomic::AtomicBool,
+    pub(crate) reftrace_print: std::sync::atomic::AtomicBool,
+}
+
+pub(crate) static RELEASE_GATES: ReleaseGates = ReleaseGates {
+    rc_shared: std::sync::atomic::AtomicBool::new(false),
+    dicts_watched: std::sync::atomic::AtomicBool::new(false),
+    funcs_watched: std::sync::atomic::AtomicBool::new(false),
+    reftrace_print: std::sync::atomic::AtomicBool::new(false),
+};
+
 /// Whether reference counts must be updated atomically: set whenever the
 /// cell bias is revoked, and also before spawning a thread that may touch
 /// objects before it registers (see [`crate::rc`]). Never cleared.
-static RC_SHARED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+static RC_SHARED: &std::sync::atomic::AtomicBool = &RELEASE_GATES.rc_shared;
 
 /// The flag [`bias_held`] reads (native code that takes references in
 /// line reads it too).
 pub(crate) fn rc_shared_flag() -> &'static std::sync::atomic::AtomicBool {
-    &RC_SHARED
+    RC_SHARED
 }
 
 /// True while one thread owns every reference count (see [`crate::rc`]).

From 0afc0d678351a951196f64b248e51fccbccbf147 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 03:50:36 -0700
Subject: [PATCH 036/173] perf: finish more calls directly and refill the GIL
 countdown in place

Compiled callers' calls between Python functions take the direct-call
protocol more often, and finish it with less work:

- An expired GIL countdown (every ~128 calls, returns and back edges)
  forced a direct return through `core_return`, unwinding the whole
  native call nest to the core loop and finishing the innermost
  activation in the general loop. With no thread waiting for the GIL,
  the checkpoint has nothing to do, so the countdown now starts over
  where it ran out (`Interpreter::countdown_out`, `h_countdown` at
  native back edges); a waiting thread still gets the hand-off.
- A direct callee's `RETURN_VALUE` hands its return back in line (a
  `direct` flag in its native state) instead of calling `h_return`, and
  the caller runs it without `Native::run`'s bookkeeping, finishing with
  a lean park (`inline_park_direct`).
- Activations the core loop's call paths push for a compiled caller's
  `CALL` or `CALL_KW` (keyword arguments, closures the site doesn't
  bind) run through the same direct run and return
  (`run_pushed_directly`), instead of a switch through the core loop.
- Direct calls bind callees with cell variables (fresh cells) and
  closures (the function's cells), and prologue `MAKE_CELL`s run
  natively, so such callees enter their native code at the start.

Per-iteration instructions (pinned seed): p_meth 4218 -> 3850, closure
probe 8305 -> 7277. Suite geomean 0.9855x the base (go 0.932, sudoku
0.906, generators_tree 0.951, tomllib_loads 0.961, chaos 0.962,
mini_interpreter 0.969, deepcopy 0.976; none worse than 1.009).
---
 crates/weavepy-vm/src/frame_jit.rs        | 318 +++++++++++++++++++---
 crates/weavepy-vm/src/gen_fast.rs         |   4 +-
 crates/weavepy-vm/src/gil.rs              |   8 +
 crates/weavepy-vm/src/lib.rs              | 128 +++++++--
 tests/regrtest/test_direct_call_shapes.py | 186 +++++++++++++
 5 files changed, 589 insertions(+), 55 deletions(-)
 create mode 100644 tests/regrtest/test_direct_call_shapes.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 4dfe687f..c0093d20 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -140,6 +140,10 @@ pub(crate) struct State {
     /// In a generator's fast step, the fast steps it is nested in (see
     /// `gen_fast`).
     pub(crate) gen_depth: u8,
+    /// A direct call's callee (see [`direct_call`]): its `RETURN_VALUE`
+    /// hands the return straight back (`DIRECT_RET`) for the caller to
+    /// finish, without [`h_return`].
+    pub(crate) direct: bool,
 }
 
 const S_LOCALS: i32 = 0;
@@ -155,6 +159,7 @@ const S_OUT: i32 = 80;
 const S_INTERP: i32 = std::mem::offset_of!(State, interp) as i32;
 const S_SW: i32 = std::mem::offset_of!(State, sw) as i32;
 const S_FRAME: i32 = std::mem::offset_of!(State, frame) as i32;
+const S_DIRECT: i32 = std::mem::offset_of!(State, direct) as i32;
 const _: () = {
     assert!(std::mem::offset_of!(State, locals) == S_LOCALS as usize);
     assert!(std::mem::offset_of!(State, stack) == S_STACK as usize);
@@ -239,6 +244,42 @@ impl Native {
         }
     }
 
+    /// [`Self::run`] for a direct call's callee, entered at `from` (its
+    /// start, unless a prologue the core loop ran came first): the
+    /// bookkeeping only for a run that didn't return.
+    #[inline(always)]
+    pub(crate) unsafe fn run_direct(&self, st: &mut State, from: usize) -> u32 {
+        let (len, cap) = (st.len, st.cap);
+        // SAFETY: the caller's contract.
+        let status = unsafe { (self.func)(st) };
+        if status != DIRECT_RET || stats::enabled() {
+            self.ran(st, from, len, cap, usize::MAX, status);
+        }
+        status
+    }
+
+    /// [`Self::run`]'s bookkeeping for a run from `from` (with the stack
+    /// `len` deep, `cap` its capacity, and `last` the last instruction).
+    #[cold]
+    #[inline(never)]
+    fn ran(&self, st: &State, from: usize, len: usize, cap: usize, last: usize, status: u32) {
+        if status == INTERP && st.pc == from && st.last == last {
+            self.note_fail(from);
+        }
+        if stats::enabled() {
+            let bail = if status != INTERP || st.pc != from {
+                ""
+            } else if cap < self.need {
+                "<cap>"
+            } else if self.depths.get(from).is_some_and(|&d| d != len as i64) {
+                "<depth>"
+            } else {
+                ""
+            };
+            stats::note(&self.name, &self.ops, from, st.pc, st.last, status, bail);
+        }
+    }
+
     /// Run from `st.pc` until an instruction the core loop must run.
     ///
     /// # Safety
@@ -265,7 +306,7 @@ impl Native {
             } else {
                 ""
             };
-            stats::note(&self.name, &self.ops, from, st.pc, status, bail);
+            stats::note(&self.name, &self.ops, from, st.pc, st.last, status, bail);
         }
         status
     }
@@ -760,13 +801,18 @@ mod stats {
         ops: &[OpCode],
         from: usize,
         at: usize,
+        last: usize,
         status: u32,
         bail: &str,
     ) {
         let op = match (status, ops.get(at)) {
             (0, Some(op)) => format!("{op:?}{bail}"),
             (0, None) => "<end>".to_owned(),
-            (super::RELOAD, _) => "<switched>".to_owned(),
+            // (Where it switched: the last instruction it ran.)
+            (super::RELOAD, _) => match ops.get(last) {
+                Some(op) => format!("<switched> at {op:?}@{last}, {:?}", ops.get(last + 1)),
+                None => "<switched>".to_owned(),
+            },
             (super::RAISED, _) => "<raised>".to_owned(),
             (super::DIRECT_RET, _) => "<direct return>".to_owned(),
             _ => "<marked>".to_owned(),
@@ -3723,6 +3769,7 @@ unsafe fn run_switched_directly(st: &mut State) -> bool {
             frame: sw.cur,
             sw: st.sw,
             gen_depth: 0,
+            direct: false,
         };
         interp.direct_calls.set(nested + 1);
         let status = native.run(&mut cst);
@@ -3848,19 +3895,22 @@ impl DirectSite {
 /// Whether a call of `f` (running `code`) with `nargs` positional
 /// arguments binds as [`Interpreter::core_call_plain`] binds it, and
 /// `code`'s native body runs from its start: that body, and how many
-/// trailing parameters take `f`'s defaults.
+/// trailing parameters take `f`'s defaults (with [`CELLS_BIT`] when the
+/// callee has cells to bind).
 fn direct_body<'a>(
     f: &crate::object::PyFunction,
     code: &'a CodeObject,
     nargs: usize,
 ) -> Option<(&'a Native, usize)> {
     let missing = (code.arg_count as usize).checked_sub(nargs)?;
+    // A callee with cell or free variables binds its cells too (see
+    // `Interpreter::bind_call_cells`).
+    let cells = !code.cellvars.is_empty() || !code.freevars.is_empty() || !f.closure.is_empty();
     if code.is_generator
         || code.is_coroutine
         || code.is_async_generator
-        || !code.cellvars.is_empty()
-        || !code.freevars.is_empty()
-        || !f.closure.is_empty()
+        || (cells && code.cellvars.is_empty() && f.lean_cells_ref(code).is_none())
+        || missing >= CELLS_BIT as usize
         || Interpreter::has_extended_params(code)
         || (missing > 0 && !Interpreter::defaults_cover(f, missing))
         || code.wire.as_ref().is_some_and(|w| w.exec_error.is_some())
@@ -3871,9 +3921,13 @@ fn direct_body<'a>(
     if ext.dispatch_len == 0 {
         return None;
     }
-    Some((ext.frame_jit.get(code.varnames.len().max(nargs))?, missing))
+    let packed = missing | if cells { CELLS_BIT as usize } else { 0 };
+    Some((ext.frame_jit.get(code.varnames.len().max(nargs))?, packed))
 }
 
+/// A [`DirectSite`]'s `missing` flag for a callee with cells to bind.
+const CELLS_BIT: u32 = 1 << 31;
+
 /// `BINARY_OP` (the `pc`th instruction of `code`) over the instances at
 /// stack slots `slot` and above it whose class's method runs alone (see
 /// `Interpreter::binop_instance_dunder`): the method called directly
@@ -3911,7 +3965,7 @@ unsafe extern "C" fn h_binop_dunder(
         at.add(2).write(at.add(1).read());
         at.add(1).write(at.read());
         at.write(Object::Function(f));
-        match direct_call(
+        match direct_call_out(
             st,
             pc,
             slot,
@@ -3998,7 +4052,7 @@ unsafe extern "C" fn h_property(
             at.add(1).write(at.read());
         }
         at.write(Object::Function(getter));
-        match direct_call(st, pc, slot, slot + 2, &mut *site, None) {
+        match direct_call_out(st, pc, slot, slot + 2, &mut *site, None) {
             None => {
                 let f = at.read();
                 if local != 0 {
@@ -4063,6 +4117,18 @@ unsafe fn attr_lane(
     }
 }
 
+/// A back edge's GIL countdown (at `countdown`) ran out: the fresh count
+/// when nobody waits for the GIL (see `Interpreter::countdown_out`), or
+/// `0` for the core loop's checkpoint.
+unsafe extern "C" fn h_countdown(countdown: *mut u32) -> u32 {
+    if crate::gil::handoff_idle() {
+        // SAFETY: the code passes its state's countdown pointer.
+        unsafe { *countdown = crate::gil::GIL_CHECK_INTERVAL };
+        return crate::gil::GIL_CHECK_INTERVAL;
+    }
+    0
+}
+
 /// [`h_call`] with a direct call ([`direct_call`]) of the plain Python
 /// function `site` caches tried first. (The native code calls `h_call`
 /// itself for any other callee, and while a site without a body waits
@@ -4090,6 +4156,21 @@ unsafe extern "C" fn h_call_direct(
     }
 }
 
+/// [`direct_call`] out of line, for the colder lanes (a property getter,
+/// an instance operator's method), which share one copy of it.
+#[inline(never)]
+unsafe fn direct_call_out(
+    st: &mut State,
+    pc: usize,
+    start: usize,
+    len: usize,
+    site: &mut DirectSite,
+    binop: Option<(BinOpKind, bool, &Rc<crate::types::TypeObject>)>,
+) -> Option<u32> {
+    // SAFETY: the caller's contract.
+    unsafe { direct_call(st, pc, start, len, site, binop) }
+}
+
 /// The `CALL` at `pc` (its operands from `start` up a `len`-deep stack) of
 /// a plain Python function whose body has native code, run straight from
 /// here: the callee's activation is bound as the core loop's arm binds it
@@ -4121,7 +4202,7 @@ unsafe fn direct_call(
         // `def` binds the same way, and code without free variables takes
         // no closure.)
         let at = Rc::as_ptr(code_rc) as usize;
-        let (native, missing) = if site.code == at && site.has_self == has_self && site.native != 0
+        let (native, packed) = if site.code == at && site.has_self == has_self && site.native != 0
         {
             (site.native, site.missing as usize)
         } else if site.alt_code == at && site.alt_has_self == has_self && site.alt_native != 0 {
@@ -4142,9 +4223,16 @@ unsafe fn direct_call(
             site.promote(code_rc, has_self, native, missing);
             (native, missing)
         };
+        let (missing, cells) = (
+            packed & !(CELLS_BIT as usize),
+            packed & CELLS_BIT as usize != 0,
+        );
         // (Another function running the same code may have rebound its
-        // defaults.)
-        if missing > 0 && !Interpreter::defaults_cover(f, missing) {
+        // defaults, or, a closure, have cells the lean binding can't take.)
+        if packed != 0
+            && ((missing > 0 && !Interpreter::defaults_cover(f, missing))
+                || (cells && code_rc.cellvars.is_empty() && f.lean_cells_ref(code_rc).is_none()))
+        {
             return None;
         }
         let native = &*(native as *const Native);
@@ -4175,12 +4263,99 @@ unsafe fn direct_call(
         caller.pc = pc as u32;
         *sw.last = st.last;
         let saved_last = sw.last;
-        let callee = interp.core_bind_plain(sw, pc, start, has_self, missing, fp, act, guard);
+        let callee = interp
+            .core_bind_plain::<true>(sw, pc, start, has_self, missing, cells, fp, act, guard);
         crate::burst_stats::note_call(crate::burst_stats::CALL_DIRECT);
         let depth = (*sw.inl).len();
-        if let Some(a) = (*sw.inl).last_mut() {
-            a.direct = true;
+        Some(direct_run::<false>(
+            st, pc, start, callee, depth, native, saved_last, nested, binop, 0,
+        ))
+    }
+}
+
+/// The activation a `CALL` at `pc` (its operands from `start`) just pushed
+/// through the core loop's call paths (`Interpreter::core_call`,
+/// `Interpreter::core_call_kw`) for a plain function, run as a direct
+/// call's ([`direct_run`]) when its code has a native body entered where
+/// it starts. `saved_last` is the caller's last-instruction slot before the
+/// switch. `None` touched nothing.
+///
+/// # Safety
+///
+/// `st` is the caller's live state, its frame synced at the call and its
+/// switch non-null.
+#[inline(never)]
+unsafe fn run_pushed_directly(
+    st: &mut State,
+    pc: usize,
+    start: usize,
+    saved_last: *mut usize,
+) -> Option<u32> {
+    // SAFETY: the caller's contract.
+    unsafe {
+        let sw = &mut *st.sw;
+        if sw.pending.is_some() || std::ptr::eq(sw.cur, st.frame) {
+            return None;
+        }
+        let inl = &mut *sw.inl;
+        let depth = inl.len();
+        let callee = sw.cur;
+        let top = inl.last_mut()?;
+        if top.gen.is_some()
+            || top.init_inst.is_some()
+            || top.binop.is_some()
+            || top.discard
+            || top.direct
+            || !std::ptr::eq(&raw const *top.frame, callee)
+        {
+            return None;
         }
+        let interp = &*st.interp;
+        let nested = interp.direct_calls.get();
+        if nested >= DIRECT_CALL_DEPTH || crate::hot_gates::loop_gen() != st.snap_gen {
+            return None;
+        }
+        let cframe = &*callee;
+        let ext = crate::code_vm_ext(&cframe.code)?;
+        let native = ext.frame_jit.get((*cframe.locals.as_ptr()).len())?;
+        let from = cframe.pc as usize;
+        if from >= ext.dispatch_len || !native.enters_at(from) {
+            return None;
+        }
+        top.direct = true;
+        Some(direct_run::<true>(
+            st, pc, start, callee, depth, native, saved_last, nested, None, from,
+        ))
+    }
+}
+
+/// Run a direct call's callee (the running activation, `callee`, at
+/// `depth`, marked `direct`) through its native code from `from`, and
+/// finish its return here: [`direct_call`]'s `Some` results. `PUSHED`
+/// for an activation the core loop's call paths bound (see
+/// [`run_pushed_directly`]), whose cells its clean return checks.
+///
+/// # Safety
+///
+/// As [`direct_call`]'s, with the callee bound and switched to.
+#[inline(always)]
+#[allow(clippy::too_many_arguments)]
+unsafe fn direct_run<const PUSHED: bool>(
+    st: &mut State,
+    pc: usize,
+    start: usize,
+    callee: *mut crate::Frame,
+    depth: usize,
+    native: &Native,
+    saved_last: *mut usize,
+    nested: u32,
+    binop: Option<(BinOpKind, bool, &Rc<crate::types::TypeObject>)>,
+    from: usize,
+) -> u32 {
+    // SAFETY: the caller's contract.
+    unsafe {
+        let interp = &mut *st.interp.cast_mut();
+        let caller = &mut *st.frame;
         let cframe = &mut *callee;
         let stack = &mut cframe.stack;
         if stack.capacity() < native.need() {
@@ -4189,9 +4364,9 @@ unsafe fn direct_call(
         let mut cst = State {
             locals: (*cframe.locals.as_ptr()).as_mut_ptr(),
             stack: stack.as_mut_ptr(),
-            len: 0,
+            len: stack.len(),
             cap: stack.capacity(),
-            pc: 0,
+            pc: from,
             last: usize::MAX,
             countdown: st.countdown,
             snap_gen: st.snap_gen,
@@ -4203,16 +4378,27 @@ unsafe fn direct_call(
             frame: callee,
             sw: st.sw,
             gen_depth: 0,
+            direct: true,
         };
         interp.direct_calls.set(nested + 1);
-        let status = native.run(&mut cst);
+        let status = native.run_direct(&mut cst, from);
         interp.direct_calls.set(nested);
         let sw = &mut *st.sw;
         let inl = &mut *sw.inl;
-        let ours = inl.len() == depth
-            && inl
-                .last()
-                .is_some_and(|a| std::ptr::eq(&raw const *a.frame, callee));
+        // (Only the callee's own `RETURN_VALUE` hands back `DIRECT_RET`,
+        // with its activation the running one.)
+        let ours = status == DIRECT_RET
+            || (inl.len() == depth
+                && inl
+                    .last()
+                    .is_some_and(|a| std::ptr::eq(&raw const *a.frame, callee)));
+        debug_assert!(
+            status != DIRECT_RET
+                || (inl.len() == depth
+                    && inl
+                        .last()
+                        .is_some_and(|a| a.direct && std::ptr::eq(&raw const *a.frame, callee)))
+        );
         // The activation goes on as any other (see `InlineAct::binop`).
         let leave = |a: &mut crate::InlineAct| {
             a.direct = false;
@@ -4220,28 +4406,35 @@ unsafe fn direct_call(
                 a.binop = Some((op, inplace, cls.clone()));
             }
         };
-        if status == DIRECT_RET && ours {
+        if status == DIRECT_RET {
             let clean = cst.len == 1
-                && inl.last().is_some_and(|a| a.act.shell.is_none())
+                && inl.last().is_some_and(|a| {
+                    a.act.shell.is_none()
+                        && (!PUSHED || cframe.code.cellvars.is_empty() || a.owns_cells)
+                })
                 && Rc::strong_count(&cframe.locals) == 1
-                && interp.gil_countdown > 2
+                && !interp.countdown_out(2)
                 && crate::hot_gates::loop_gen() == st.snap_gen;
             if clean {
                 // `core_return`'s clean return, with the caller known.
                 let v = cst.stack.read();
                 cframe.stack.set_len(0);
-                let mut done = inl.pop().expect("checked above");
+                let mut done = inl.pop().unwrap_unchecked();
                 if depth == sw.entry_depth {
                     sw.entry_dead = true;
                 }
-                done.clean = true;
                 crate::release_locals(&mut *cframe.locals.as_ptr());
                 drop(done.guard.take());
                 interp.lean_pending_exit(done.caller_pending);
                 // SAFETY: moved out once; the parked slot treats the field
                 // as stale (see `inline_deliver`).
                 let callable = std::ptr::read(&raw const done.callable);
-                interp.inline_park_clean(done);
+                if done.owns_cells {
+                    done.clean = true;
+                    interp.inline_park_clean(done);
+                } else {
+                    interp.inline_park_direct(done);
+                }
                 st.stack.add(start).write(v);
                 caller.stack.set_len(start + 1);
                 match callable {
@@ -4264,10 +4457,10 @@ unsafe fn direct_call(
                 }
                 *sw.last = pc;
                 if sw.pending.is_some() {
-                    return Some(RELOAD);
+                    return RELOAD;
                 }
                 st.len = start + 1;
-                return Some(0);
+                return 0;
             }
             // Any other return is the core loop's arm's (as `h_return`'s).
             cframe.stack.set_len(cst.len);
@@ -4292,9 +4485,9 @@ unsafe fn direct_call(
                 && caller.stack.len() == start + 1
             {
                 st.len = start + 1;
-                return Some(0);
+                return 0;
             }
-            return Some(RELOAD);
+            return RELOAD;
         }
         // The callee stopped partway: its state, synced as the core loop
         // syncs it after a native run, for the core loop to go on with.
@@ -4307,7 +4500,7 @@ unsafe fn direct_call(
         } else if let Some(e) = cst.err.take() {
             sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Raised(e)));
         }
-        Some(RELOAD)
+        RELOAD
     }
 }
 
@@ -4466,6 +4659,7 @@ unsafe extern "C" fn h_call(
             }
             sync(st);
             let sw = &mut *st.sw;
+            let saved_last = sw.last;
             let interp = &mut *st.interp.cast_mut();
             let switched = if python == 1 {
                 interp.core_call(sw, pc) || interp.core_gen_call(sw, pc)
@@ -4478,6 +4672,11 @@ unsafe extern "C" fn h_call(
             // A compiled callee runs from here, and its return continues
             // this code natively (anything else leaves for the core loop).
             if switched && sw.pending.is_none() && !std::ptr::eq(sw.cur, st.frame) {
+                if python == 1 {
+                    if let Some(r) = run_pushed_directly(st, pc, start, saved_last) {
+                        return r;
+                    }
+                }
                 if run_callee_directly(st, start) {
                     st.len = start + 1;
                     return 0;
@@ -4843,9 +5042,17 @@ unsafe extern "C" fn h_call_kw(st: *mut State, code: *const CodeObject, pc: u64,
             frame.pc = pc as u32;
             let sw = &mut *st.sw;
             *sw.last = st.last;
+            let saved_last = sw.last;
             let interp = &mut *st.interp.cast_mut();
-            if !interp.core_call_kw(sw, pc) && sw.pending.is_none() {
-                sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+            if !interp.core_call_kw(sw, pc) {
+                if sw.pending.is_none() {
+                    sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+                }
+                return RELOAD;
+            }
+            // A compiled callee runs from here, as for `CALL`.
+            if let Some(r) = run_pushed_directly(st, pc, start, saved_last) {
+                return r;
             }
             return RELOAD;
         };
@@ -5776,6 +5983,8 @@ fn native_at(code: &CodeObject, pc: usize) -> bool {
         OpCode::BuildMap => (1..=8).contains(&ins.arg),
         OpCode::UnaryOp => ins.arg <= 3,
         OpCode::RaiseVarargs => matches!(ins.arg, 1 | 2),
+        // A prologue `MAKE_CELL`: the activation's cells are built with it.
+        OpCode::MakeCell => prologue_cell(code, pc),
         // (A slot shared with a cell saves the cell itself.)
         OpCode::LoadFastAndClear => {
             Interpreter::shared_cell_index(code, ins.arg as usize).is_none()
@@ -5784,6 +5993,15 @@ fn native_at(code: &CodeObject, pc: usize) -> bool {
     }
 }
 
+/// Whether the `MAKE_CELL` at `pc` is in the code's prologue, which every
+/// activation's cells satisfy as they are built (the core loop runs it as
+/// a no-op too); a later one is an inlined comprehension's fresh cell.
+fn prologue_cell(code: &CodeObject, pc: usize) -> bool {
+    code.instructions[..pc]
+        .iter()
+        .all(|i| matches!(i.op, OpCode::MakeCell | OpCode::CopyFreeVars))
+}
+
 impl<'a> Lower<'a> {
     #[allow(clippy::too_many_arguments)]
     fn new(
@@ -6557,6 +6775,7 @@ impl<'a> Lower<'a> {
         }
         let cont = match ins.op {
             OpCode::Nop | OpCode::NotTaken | OpCode::Resume | OpCode::CopyFreeVars => true,
+            OpCode::MakeCell if prologue_cell(self.code, pc) => true,
             OpCode::LoadFast | OpCode::LoadFastBorrow => self.load_fast(pc, ins.arg),
             OpCode::LoadConst if (ins.arg as usize) < self.ext.objects.len() => {
                 self.push(Item::Const(ins.arg));
@@ -6734,9 +6953,22 @@ impl<'a> Lower<'a> {
                 let g = self.b.ins().load(types::I64, FLAGS, gen_addr, 0);
                 let snap = self.b.ins().load(types::I64, FLAGS, self.st, S_SNAP);
                 let moved = self.b.ins().icmp(IntCC::NotEqual, g, snap);
-                let trip = self.b.ins().bor(low, moved);
                 let out = self.exit_with(pc, &[], INTERP);
-                self.branch_out(trip, out);
+                self.branch_out(moved, out);
+                // A countdown that ran out with nobody waiting for the GIL
+                // starts over here (see `Interpreter::countdown_out`).
+                let refill = self.b.create_block();
+                let go = self.b.create_block();
+                self.b.append_block_param(go, types::I32);
+                self.b.set_cold_block(refill);
+                self.b.ins().brif(low, refill, &[], go, &[c.into()]);
+                self.b.switch_to_block(refill);
+                let n = self
+                    .call(h_countdown as *const () as usize, &[countdown], true)
+                    .expect("returns");
+                self.b.ins().brif(n, go, &[n.into()], out, &[]);
+                self.b.switch_to_block(go);
+                let c = self.b.block_params(go)[0];
                 let c1 = self.b.ins().iadd_imm(c, -1);
                 self.b.ins().store(FLAGS, c1, countdown, 0);
                 self.set_last(pc);
@@ -7054,6 +7286,18 @@ impl<'a> Lower<'a> {
                 self.store_last();
                 let pcv = self.b.ins().iconst(types::I64, pc as i64);
                 let len = self.b.ins().iconst(types::I64, self.depth as i64);
+                // A direct call's callee hands its return to the caller as
+                // it stands (`h_return`'s `DIRECT_RET`, in line).
+                let direct = self.b.ins().uload8(types::I32, FLAGS, self.st, S_DIRECT);
+                let handed = self.b.create_block();
+                let other = self.b.create_block();
+                self.b.ins().brif(direct, handed, &[], other, &[]);
+                self.b.switch_to_block(handed);
+                self.b.ins().store(FLAGS, len, self.st, S_LEN);
+                self.b.ins().store(FLAGS, pcv, self.st, S_PC);
+                let r = self.b.ins().iconst(types::I32, i64::from(DIRECT_RET));
+                self.b.ins().return_(&[r]);
+                self.b.switch_to_block(other);
                 let r = self
                     .call(h_return as *const () as usize, &[self.st, pcv, len], true)
                     .expect("returns");
diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index c89ea35a..81cf3795 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -515,6 +515,7 @@ impl Interpreter {
                         frame: frame_ptr,
                         sw: std::ptr::null_mut(),
                         gen_depth: depth,
+                        direct: false,
                     });
                     nst.len = len;
                     nst.pc = pc;
@@ -683,7 +684,7 @@ impl Interpreter {
                 OpCode::JumpForward => pc += 1 + ins.arg as usize,
                 OpCode::JumpBackward => {
                     // The back edge is the eval-breaker (as the core loop's).
-                    if self.gil_countdown <= 1 || crate::hot_gates::loop_gen() != snap_gen {
+                    if self.countdown_out(1) || crate::hot_gates::loop_gen() != snap_gen {
                         break None;
                     }
                     self.gil_countdown -= 1;
@@ -857,6 +858,7 @@ impl Interpreter {
             frame: frame_ptr,
             sw: std::ptr::null_mut(),
             gen_depth: depth,
+            direct: false,
         };
         // SAFETY: the body's activation state, as `gen_fast_step` holds it.
         let status = unsafe { native.run(&mut st) };
diff --git a/crates/weavepy-vm/src/gil.rs b/crates/weavepy-vm/src/gil.rs
index 54a07130..41a2749c 100644
--- a/crates/weavepy-vm/src/gil.rs
+++ b/crates/weavepy-vm/src/gil.rs
@@ -892,6 +892,14 @@ pub fn yield_checkpoint() {
     maybe_yield_gil();
 }
 
+/// Whether a checkpoint would return at once: no thread waits for the
+/// GIL and no waiter's clock is armed (see [`maybe_yield_gil`]). An
+/// expired countdown then starts over where it ran out.
+#[inline]
+pub(crate) fn handoff_idle() -> bool {
+    global_gil_ref().breaker.waiter_count() == 0 && !WAITER_CLOCK_ARMED.load(Ordering::Relaxed)
+}
+
 /// Hand the GIL to a waiting thread, if any. Drops the calling
 /// thread's guard stack (releasing the lock), spins briefly so a
 /// waiter can take it, then re-acquires. No-op when nobody is
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 20ecac65..f60165ca 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -7681,6 +7681,29 @@ impl Interpreter {
         }
     }
 
+    /// Whether the GIL countdown is down to `floor` with a hand-off to
+    /// make: the quiet paths' calls, returns and back edges then yield to
+    /// the outer loop's checkpoint. A countdown that runs out with no
+    /// thread waiting starts over in place instead (the checkpoint would
+    /// do nothing else), so a busy single thread doesn't unwind its whole
+    /// native call nest every 128 calls.
+    #[inline(always)]
+    pub(crate) fn countdown_out(&mut self, floor: u32) -> bool {
+        self.gil_countdown <= floor && !self.countdown_refill()
+    }
+
+    /// [`Self::countdown_out`]'s cold half: `true` when the countdown
+    /// started over.
+    #[cold]
+    #[inline(never)]
+    pub(crate) fn countdown_refill(&mut self) -> bool {
+        if crate::gil::handoff_idle() {
+            self.gil_countdown = crate::gil::GIL_CHECK_INTERVAL;
+            return true;
+        }
+        false
+    }
+
     /// After the quiet loop caught an exception: whether the activation
     /// yields as the outer loop's per-instruction prologue would (the GIL
     /// countdown, a hot gate). Code that only calls and catches, such as
@@ -7688,7 +7711,7 @@ impl Interpreter {
     /// never let another thread or a signal in.
     #[inline]
     fn quiet_caught_yields(&mut self, snap_gen: u64) -> bool {
-        if self.gil_countdown <= 2 {
+        if self.countdown_out(2) {
             return true;
         }
         self.gil_countdown -= 1;
@@ -7722,7 +7745,7 @@ impl Interpreter {
                 // lean paths below.
                 QuietEntry::Returned { cur_pc } => {
                     *last = cur_pc;
-                    if self.gil_countdown <= 2 {
+                    if self.countdown_out(2) {
                         break 'run QuietExit::Yield;
                     }
                     self.gil_countdown -= 1;
@@ -7892,7 +7915,7 @@ impl Interpreter {
                             });
                         }
                         *last = cur_pc;
-                        if self.gil_countdown <= 2 {
+                        if self.countdown_out(2) {
                             break 'run QuietExit::Yield;
                         }
                         self.gil_countdown -= 1;
@@ -7976,7 +7999,7 @@ impl Interpreter {
                             });
                         }
                         *last = cur_pc;
-                        if self.gil_countdown <= 2 {
+                        if self.countdown_out(2) {
                             break 'run QuietExit::Yield;
                         }
                         self.gil_countdown -= 1;
@@ -8031,7 +8054,7 @@ impl Interpreter {
                 }
                 *last = cur_pc;
                 // The outer loop's per-instruction prologue, in its order.
-                if self.gil_countdown <= 2 {
+                if self.countdown_out(2) {
                     break 'run QuietExit::Yield;
                 }
                 self.gil_countdown -= 1;
@@ -8289,6 +8312,39 @@ impl Interpreter {
         }
     }
 
+    /// [`Self::inline_park_clean`] for a direct call's activation (see
+    /// `frame_jit::direct_call`) after its clean return: it never owns
+    /// cells, and of the fields the park resets only `direct` was set
+    /// (its guard is gone already).
+    #[inline(always)]
+    pub(crate) fn inline_park_direct(&mut self, mut act: Box<InlineAct>) {
+        debug_assert!(
+            act.frame.stack.is_empty()
+                && !act.owns_cells
+                && act.binop.is_none()
+                && !act.discard
+                && act.guard.is_none()
+                && act.init_inst.is_none()
+                && act.act.shell.is_none()
+        );
+        // SAFETY: as `inline_park`: the code handle is owned while active;
+        // a function holds the code too, so this is rarely its last owner.
+        unsafe {
+            let code = std::ptr::read(&raw const act.frame.code);
+            if crate::rc::try_release_shared(Rc::as_ptr(&code)) {
+                std::mem::forget(code);
+            } else {
+                drop(code);
+            }
+        }
+        act.parked = true;
+        act.direct = false;
+        act.caller_pending = None;
+        if self.inline_pool.len() < INLINE_POOL_CAP {
+            push_fast(&mut self.inline_pool, act);
+        }
+    }
+
     /// `FOR_ITER` over a suspended generator at `pc` of `frame`, as an
     /// inline activation: the generator's boxed frame runs in place (the
     /// `generator_send_lean` shape, without a nested native activation).
@@ -11361,6 +11417,7 @@ impl Interpreter {
                             frame: sw.cur,
                             sw: std::ptr::from_mut(sw),
                             gen_depth: 0,
+                            direct: false,
                         });
                         nst.len = len;
                         nst.pc = pc;
@@ -12495,7 +12552,7 @@ impl Interpreter {
                     OpCode::JumpBackward => {
                         // The back edge is the burst's eval-breaker (see
                         // `leaf_burst_slow`).
-                        if self.gil_countdown <= 1 || crate::hot_gates::loop_gen() != snap_gen {
+                        if self.countdown_out(1) || crate::hot_gates::loop_gen() != snap_gen {
                             break Some(CoreExit::Stop(LeafStop::Breaker));
                         }
                         #[cfg(feature = "jit")]
@@ -15395,7 +15452,11 @@ impl Interpreter {
             return false;
         };
         // SAFETY: as above; the checks above hold.
-        unsafe { self.core_bind_plain(sw, pc, callee_slot, has_self, missing, fp, act, guard) };
+        unsafe {
+            self.core_bind_plain::<false>(
+                sw, pc, callee_slot, has_self, missing, false, fp, act, guard,
+            )
+        };
         true
     }
 
@@ -15412,7 +15473,8 @@ impl Interpreter {
     /// pooled slot `act`'s locals, and `act`, bound to the plain function
     /// `fp` (with exactly its arity) under the recursion `guard`, is
     /// pushed and made the running activation. The last `missing`
-    /// parameters take `fp`'s positional defaults.
+    /// parameters take `fp`'s positional defaults. `DIRECT` marks the
+    /// activation a direct call's (see `frame_jit::direct_call`).
     ///
     /// # Safety
     ///
@@ -15421,13 +15483,14 @@ impl Interpreter {
     /// `missing`, see [`Self::defaults_cover`]).
     #[inline(always)]
     #[allow(clippy::too_many_arguments)]
-    unsafe fn core_bind_plain(
+    unsafe fn core_bind_plain<const DIRECT: bool>(
         &mut self,
         sw: &mut CoreSwitch,
         pc: usize,
         callee_slot: usize,
         has_self: bool,
         missing: usize,
+        cells: bool,
         fp: *const crate::object::PyFunction,
         mut act: Box<InlineAct>,
         guard: crate::recursion::Guard,
@@ -15490,9 +15553,13 @@ impl Interpreter {
             std::ptr::write(&raw mut fr.globals, borrowed_rc(&(*fp).globals));
             std::ptr::write(&raw mut fr.builtins, borrowed_rc(&(*fp).builtins));
             std::ptr::write(&raw mut act.callable, callable);
+            if cells {
+                Self::bind_call_cells(&mut act, &*fp);
+            }
         }
         act.frame.pc = 0;
         act.parked = false;
+        act.direct = DIRECT;
         act.call_pc = pc;
         // The caller waits on the pending list (see `lean_pending_enter`).
         act.caller_pending = self.core_pending_enter(sw, frame, pc);
@@ -15520,6 +15587,33 @@ impl Interpreter {
         callee
     }
 
+    /// [`Self::core_bind_plain`]'s cells for a callee with cell or free
+    /// variables (a direct call's, see `frame_jit::direct_body`): fresh
+    /// cells of its own (its parameters that are cells move into them, as
+    /// `inline_bind` binds them), or its closure's lean cells, which `f`
+    /// holds (see `PyFunction::lean_cells_ref`).
+    ///
+    /// # Safety
+    ///
+    /// `act` is freshly bound to `f` (its locals filled, its cells handle
+    /// a stale copy, overwritten without a drop).
+    #[cold]
+    #[inline(never)]
+    unsafe fn bind_call_cells(act: &mut InlineAct, f: &PyFunction) {
+        // SAFETY: the caller's contract; the code outlives the binding.
+        unsafe {
+            let code: &CodeObject = &*Rc::as_ptr(&act.frame.code);
+            if code.cellvars.is_empty() {
+                let cells = f.lean_cells_ref(code).expect("checked by the call");
+                std::ptr::write(&raw mut act.frame.cells, borrowed_rc(cells));
+            } else {
+                let cells = fresh_cells(f, code, &mut *act.frame.locals.as_ptr());
+                std::ptr::write(&raw mut act.frame.cells, cells);
+                act.owns_cells = true;
+            }
+        }
+    }
+
     /// A bound method over a plain function with an empty self slot,
     /// called at `pc` of `frame`, becomes that function with the receiver
     /// as self (CPython's `CALL_BOUND_METHOD_EXACT_ARGS`), so the cached
@@ -20545,7 +20639,7 @@ impl Interpreter {
                 .as_deref()
                 .is_some_and(|s| !self.shell_pops_quietly(s, top.exc_depth))
             || Self::collapse_flushed(top)
-            || self.gil_countdown <= 2
+            || self.countdown_out(2)
             || crate::hot_gates::loop_gen() != snap_gen
         {
             return false;
@@ -20632,7 +20726,7 @@ impl Interpreter {
             }
             // SAFETY: as above.
             unsafe { push_fast(&mut (*cframe).stack, v) };
-            if self.gil_countdown <= 2 {
+            if self.countdown_out(2) {
                 sw.pending = Some(CoreExit::Stop(LeafStop::Breaker));
                 return true;
             }
@@ -20672,7 +20766,7 @@ impl Interpreter {
         // (a `StopIteration`), finishes through `quiet_run`.
         if top.act.shell.is_some()
             || top.exhaust_arg == GEN_NEXT_CALL
-            || self.gil_countdown <= 2
+            || self.countdown_out(2)
             || crate::hot_gates::loop_gen() != snap_gen
         {
             return false;
@@ -20738,7 +20832,7 @@ impl Interpreter {
         // The consumer's `QuietEntry::Returned` protocol (`quiet_frame`).
         // SAFETY: as above.
         unsafe { *sw.last = call_pc };
-        if self.gil_countdown <= 2 {
+        if self.countdown_out(2) {
             sw.pending = Some(CoreExit::Stop(LeafStop::Breaker));
             return true;
         }
@@ -20785,7 +20879,7 @@ impl Interpreter {
         if top.act.shell.as_deref().is_some_and(|s| {
             !self.shell_pops_quietly(s, top.exc_depth)
                 && (self.exc_info_len() > top.exc_depth || crate::trace::any_observers_active())
-        }) || self.gil_countdown <= 2
+        }) || self.countdown_out(2)
             || crate::hot_gates::loop_gen() != snap_gen
         {
             return false;
@@ -20860,7 +20954,7 @@ impl Interpreter {
             // below.
             // SAFETY: as above.
             unsafe { *sw.last = call_pc };
-            if self.gil_countdown <= 2 {
+            if self.countdown_out(2) {
                 sw.pending = Some(CoreExit::Stop(LeafStop::Breaker));
                 return true;
             }
@@ -20914,7 +21008,7 @@ impl Interpreter {
                 // SAFETY: as above.
                 unsafe { *sw.last = cur_pc };
                 // The caller's `Returned` entry protocol (`quiet_frame`).
-                if self.gil_countdown <= 2 {
+                if self.countdown_out(2) {
                     sw.pending = Some(CoreExit::Stop(LeafStop::Breaker));
                     return true;
                 }
@@ -22853,7 +22947,7 @@ impl Interpreter {
                     // hand-off cadence, every loop-generation input
                     // (signals, async exceptions, observers, pending
                     // finalizers), and tier-2 heat are all polled here.
-                    if self.gil_countdown <= 1 || crate::hot_gates::loop_gen() != snap_gen {
+                    if self.countdown_out(1) || crate::hot_gates::loop_gen() != snap_gen {
                         stop = LeafStop::Breaker;
                         break;
                     }
diff --git a/tests/regrtest/test_direct_call_shapes.py b/tests/regrtest/test_direct_call_shapes.py
new file mode 100644
index 00000000..9a9ab19d
--- /dev/null
+++ b/tests/regrtest/test_direct_call_shapes.py
@@ -0,0 +1,186 @@
+"""Compiled callers call more shapes directly, with the same semantics.
+
+A compiled function's call of another compiled function runs the callee
+from the caller's call helper and finishes its return there. That now
+covers callees with cell variables (their own fresh cells), closures (the
+function's cells), and calls with keyword arguments; a callee's
+`RETURN_VALUE` hands its return straight back; and an eval-breaker
+countdown that runs out with no thread waiting for the GIL starts over in
+place. Each loop below runs long enough for its functions to compile.
+"""
+
+import sys
+import threading
+import traceback
+
+N = 60000
+
+
+def make_adder(k):
+    def add(x):
+        return x + k
+
+    return add
+
+
+def make_counter():
+    n = 0
+
+    def bump(by):
+        nonlocal n
+        n += by
+        return n
+
+    return bump
+
+
+def cell_param(a, b):
+    def get():
+        return a + b
+
+    a += 1
+    return get()
+
+
+def closures():
+    total = 0
+    bump = make_counter()
+    for i in range(N):
+        add = make_adder(i)
+        total += add(1) + add(2)
+        total += cell_param(i, 1)
+        bump(2)
+    expect = sum(2 * i + 3 for i in range(N)) + sum(i + 2 for i in range(N))
+    assert total == expect, (total, expect)
+    assert bump(0) == 2 * N
+
+
+def kw_callee(a, b=1, *, c=2):
+    return a * 100 + b * 10 + c
+
+
+def kw_positional(a, scale=1):
+    return a * scale
+
+
+def kw_calls():
+    t = 0
+    for i in range(N):
+        t += kw_positional(i, scale=2)
+        t += kw_callee(1, b=2, c=3)
+    assert t == sum(2 * i for i in range(N)) + 123 * N, t
+
+
+class Dying:
+    log = []
+
+    def __init__(self, name):
+        self.name = name
+
+    def __del__(self):
+        Dying.log.append(self.name)
+
+
+def holds_cell(d):
+    def peek():
+        return d.name
+
+    return peek()
+
+
+def del_timing():
+    for i in range(N):
+        Dying.log.clear()
+        name = holds_cell(Dying(i))
+        # The argument dies with the callee's cells, before the caller goes on.
+        assert Dying.log == [i], (i, Dying.log)
+        assert name == i
+
+
+def frames_through_closures():
+    def inner(depth):
+        f = sys._getframe()
+        return (f.f_code.co_name, f.f_back.f_code.co_name, depth)
+
+    def outer(depth):
+        return inner(depth)
+
+    for i in range(N):
+        assert outer(i) == ("inner", "outer", i)
+
+
+def raising_closure():
+    k = 3
+
+    def boom(x):
+        if x == N - 1:
+            raise ValueError(x + k)
+        return x
+
+    def call(x):
+        return boom(x)
+
+    for i in range(N - 1):
+        assert call(i) == i
+    try:
+        call(N - 1)
+    except ValueError as e:
+        names = [f.name for f in traceback.extract_tb(e.__traceback__)]
+        assert names[-2:] == ["call", "boom"], names
+        assert e.args == (N - 1 + k,)
+    else:
+        raise AssertionError("no raise")
+
+
+def recursion_limit():
+    def down(n):
+        def step():
+            return n
+
+        if n == 0:
+            return step()
+        return down(n - 1) + 1
+
+    for _ in range(200):
+        assert down(20) == 20
+    old = sys.getrecursionlimit()
+    sys.setrecursionlimit(200)
+    try:
+        down(500)
+    except RecursionError:
+        pass
+    else:
+        raise AssertionError("no RecursionError")
+    finally:
+        sys.setrecursionlimit(old)
+
+
+def busy_thread_hand_off():
+    # A compiled loop of direct calls that never blocks must still let a
+    # thread waiting for the GIL in.
+    flag = []
+
+    def worker():
+        flag.append(1)
+
+    def spin(i):
+        return i + 1
+
+    t = threading.Thread(target=worker)
+    t.start()
+    i = 0
+    while not flag:
+        i = spin(i)
+        if i > 50_000_000:
+            raise AssertionError("the waiting thread never ran")
+    t.join()
+
+
+closures()
+kw_calls()
+del_timing()
+frames_through_closures()
+raising_closure()
+recursion_limit()
+busy_thread_hand_off()
+print("direct call shapes: ok")

From 42bff230b315d81a387831a6373ba0170aa83f08 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 04:19:39 -0700
Subject: [PATCH 037/173] perf: compile compiled code's callees sooner and
 finish constructors in line

A compiled caller's call of a function without a native body switches to
the core loop and back, a round trip that costs about as much as
interpreting the callee's whole body. Such calls now count the callee's
length toward its compile (`Slot::warm_switched`) where a call from the
core loop counts a tenth, so small methods that only compiled code calls
(go's `ZobristHash.add`, `EmptySet.add`) compile within a few thousand
calls instead of twenty thousand and take the direct lanes from then on.

A constructor's `__init__` activation returning `None` now finishes on
the clean return paths (`core_return`'s and the direct run's), handing
the caller the instance, instead of the general `inline_finish` /
`inline_deliver` epilogue; constructors a compiled caller pushes run
through the direct run too. The frame JIT's switch statistics name the
activation switched to.

Suite geomean 0.9745x the base (was 0.9855x): tomllib_loads 0.872,
sudoku 0.899, go 0.904, raytrace 0.913, chaos 0.917, generators_tree
0.950, mini_interpreter 0.951; none worse than 1.009.
---
 crates/weavepy-vm/src/frame_jit.rs | 71 +++++++++++++++++++++++-------
 crates/weavepy-vm/src/lib.rs       | 38 +++++++++++++++-
 2 files changed, 92 insertions(+), 17 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index c0093d20..ef7f0841 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -276,7 +276,7 @@ impl Native {
             } else {
                 ""
             };
-            stats::note(&self.name, &self.ops, from, st.pc, st.last, status, bail);
+            stats::note(&self.name, &self.ops, from, st.pc, st.last, st.sw, status, bail);
         }
     }
 
@@ -306,7 +306,7 @@ impl Native {
             } else {
                 ""
             };
-            stats::note(&self.name, &self.ops, from, st.pc, st.last, status, bail);
+            stats::note(&self.name, &self.ops, from, st.pc, st.last, st.sw, status, bail);
         }
         status
     }
@@ -371,6 +371,25 @@ impl Slot {
         }
     }
 
+    /// Count a compiled caller's call into an activation of this code that
+    /// had to switch to the core loop for want of a native body: the round
+    /// trip costs about what interpreting the whole body does, so it
+    /// counts the code's length (a call the core loop makes counts a
+    /// tenth of it), and a callee of compiled code compiles sooner.
+    #[inline]
+    pub(crate) fn warm_switched(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize) {
+        if self.native.get().is_some() {
+            return;
+        }
+        let n = code.instructions.len() as u32;
+        let h = self.heat.load(Ordering::Relaxed).saturating_add(n);
+        self.heat.store(h, Ordering::Relaxed);
+        if h >= hot().saturating_mul(n) {
+            self.heat.store(0, Ordering::Relaxed);
+            self.try_compile(code, ext, nlocals, Heat::Call);
+        }
+    }
+
     #[cold]
     #[inline(never)]
     fn try_compile(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize, at: Heat) {
@@ -802,17 +821,28 @@ mod stats {
         from: usize,
         at: usize,
         last: usize,
+        sw: *mut crate::CoreSwitch,
         status: u32,
         bail: &str,
     ) {
         let op = match (status, ops.get(at)) {
             (0, Some(op)) => format!("{op:?}{bail}"),
             (0, None) => "<end>".to_owned(),
-            // (Where it switched: the last instruction it ran.)
-            (super::RELOAD, _) => match ops.get(last) {
-                Some(op) => format!("<switched> at {op:?}@{last}, {:?}", ops.get(last + 1)),
-                None => "<switched>".to_owned(),
-            },
+            // (Where it switched: the last instruction it ran, and the
+            // activation it switched to.)
+            (super::RELOAD, _) => {
+                // SAFETY: a live switch's running activation.
+                let to = (!sw.is_null())
+                    .then(|| unsafe {
+                        let cur: &crate::Frame = &*(*sw).cur;
+                        cur.code.qualname.clone()
+                    })
+                    .unwrap_or_default();
+                match ops.get(last) {
+                    Some(op) => format!("<switched> at {op:?}@{last}, {:?} to {to}", ops.get(last + 1)),
+                    None => "<switched>".to_owned(),
+                }
+            }
             (super::RAISED, _) => "<raised>".to_owned(),
             (super::DIRECT_RET, _) => "<direct return>".to_owned(),
             _ => "<marked>".to_owned(),
@@ -4302,7 +4332,6 @@ unsafe fn run_pushed_directly(
         let callee = sw.cur;
         let top = inl.last_mut()?;
         if top.gen.is_some()
-            || top.init_inst.is_some()
             || top.binop.is_some()
             || top.discard
             || top.direct
@@ -4317,7 +4346,11 @@ unsafe fn run_pushed_directly(
         }
         let cframe = &*callee;
         let ext = crate::code_vm_ext(&cframe.code)?;
-        let native = ext.frame_jit.get((*cframe.locals.as_ptr()).len())?;
+        let nlocals = (*cframe.locals.as_ptr()).len();
+        let Some(native) = ext.frame_jit.get(nlocals) else {
+            ext.frame_jit.warm_switched(&cframe.code, ext, nlocals);
+            return None;
+        };
         let from = cframe.pc as usize;
         if from >= ext.dispatch_len || !native.enters_at(from) {
             return None;
@@ -4410,19 +4443,29 @@ unsafe fn direct_run<const PUSHED: bool>(
             let clean = cst.len == 1
                 && inl.last().is_some_and(|a| {
                     a.act.shell.is_none()
-                        && (!PUSHED || cframe.code.cellvars.is_empty() || a.owns_cells)
+                        && (!PUSHED
+                            || ((cframe.code.cellvars.is_empty() || a.owns_cells)
+                                && (a.init_inst.is_none()
+                                    || matches!(&*cst.stack, Object::None))))
                 })
                 && Rc::strong_count(&cframe.locals) == 1
                 && !interp.countdown_out(2)
                 && crate::hot_gates::loop_gen() == st.snap_gen;
             if clean {
-                // `core_return`'s clean return, with the caller known.
+                // `core_return`'s clean return, with the caller known (a
+                // constructor's `__init__` returned `None`: the caller takes
+                // the instance).
                 let v = cst.stack.read();
                 cframe.stack.set_len(0);
                 let mut done = inl.pop().unwrap_unchecked();
                 if depth == sw.entry_depth {
                     sw.entry_dead = true;
                 }
+                let v = if PUSHED {
+                    done.init_inst.take().unwrap_or(v)
+                } else {
+                    v
+                };
                 crate::release_locals(&mut *cframe.locals.as_ptr());
                 drop(done.guard.take());
                 interp.lean_pending_exit(done.caller_pending);
@@ -4672,10 +4715,8 @@ unsafe extern "C" fn h_call(
             // A compiled callee runs from here, and its return continues
             // this code natively (anything else leaves for the core loop).
             if switched && sw.pending.is_none() && !std::ptr::eq(sw.cur, st.frame) {
-                if python == 1 {
-                    if let Some(r) = run_pushed_directly(st, pc, start, saved_last) {
-                        return r;
-                    }
+                if let Some(r) = run_pushed_directly(st, pc, start, saved_last) {
+                    return r;
                 }
                 if run_callee_directly(st, start) {
                     st.len = start + 1;
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index f60165ca..f1270e99 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -14859,6 +14859,33 @@ impl Interpreter {
         }
     }
 
+    /// [`Self::core_pending_enter`] with its commonest case in line: the
+    /// caller an inline activation other than the burst's entry, with no
+    /// shell yet (the call paths' callers).
+    #[inline(always)]
+    fn core_pending_enter_fast(
+        &mut self,
+        sw: &mut CoreSwitch,
+        frame: &mut Frame,
+        pc: usize,
+    ) -> Option<usize> {
+        // SAFETY: as `core_pending_enter`.
+        unsafe {
+            let inl: &mut Vec<Box<InlineAct>> = &mut *sw.inl;
+            let depth = inl.len();
+            if depth != 0 && (depth != sw.entry_depth || sw.entry_dead) {
+                let caller: &mut InlineAct = &mut inl[depth - 1];
+                if caller.act.shell.is_none() {
+                    caller.act.frame = std::ptr::from_mut(frame);
+                    let addr = std::ptr::from_mut::<LeanAct>(&mut caller.act) as usize;
+                    push_fast(&mut self.lean_pending, addr);
+                    return Some(addr);
+                }
+            }
+        }
+        self.core_pending_enter(sw, frame, pc)
+    }
+
     /// Put `sw`'s running activation, `frame` (synced, suspended at the
     /// call at `pc`), on the pending list (see [`Self::lean_pending_enter`])
     /// while code it calls runs: anything that needs the whole spine then
@@ -15562,7 +15589,7 @@ impl Interpreter {
         act.direct = DIRECT;
         act.call_pc = pc;
         // The caller waits on the pending list (see `lean_pending_enter`).
-        act.caller_pending = self.core_pending_enter(sw, frame, pc);
+        act.caller_pending = self.core_pending_enter_fast(sw, frame, pc);
         act.exc_depth = self.exc_info_len();
         act.guard = Some(guard);
         // As `inline_bind_cells`: a hot body is compiled once, so native
@@ -20914,7 +20941,14 @@ impl Interpreter {
             && Rc::strong_count(&frame.locals) == 1;
         let mut tmp = None;
         let (cframe, clast, cshell, entry);
-        if clean && done.init_inst.is_none() && done.binop.is_none() && !done.discard {
+        if clean
+            && (done.init_inst.is_none() || matches!(v, Object::None))
+            && done.binop.is_none()
+            && !done.discard
+        {
+            // A constructor's `__init__` returned `None`: the caller takes
+            // the instance (`inline_deliver`'s epilogue).
+            let v = done.init_inst.take().unwrap_or(v);
             done.clean = !had_shell;
             // SAFETY: as above.
             // SAFETY: as above.

From a0b4dbe0e7b0dcbde09a1799577500f66713558f Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 05:19:11 -0700
Subject: [PATCH 038/173] perf: run CALL_FUNCTION_EX from compiled code through
 the call lanes

A compiled function's `CALL_FUNCTION_EX` left its native code for the
core loop, which ran the call and then re-entered the caller. It now
goes through `h_call_ex`, which runs the core loop's arm in place
(`core_call_ex`, else its lane) and, when the call pushed a compiled
callee, runs it as a direct call (`run_pushed_directly`), so `f(*args)`
and `max(*xs)` no longer cost an exit and a re-entry. The frame JIT's
statistics count why a pushed call wasn't run directly.

A generator's fast steps keep their eval-breaker stop at an expired GIL
countdown (the previous change's refill there kept tier 2 from ever
seeing such a generator; `jit_generator_parked_close_and_throw` caught
it); the caller services that stop in place already.

Suite geomean within noise of the previous change (1.003x; mini_interpreter's
`Call.eval` now returns directly).
---
 crates/weavepy-vm/src/frame_jit.rs        | 95 ++++++++++++++++++++---
 crates/weavepy-vm/src/gen_fast.rs         |  2 +-
 tests/regrtest/test_direct_call_shapes.py | 32 ++++++++
 3 files changed, 117 insertions(+), 12 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index ef7f0841..c1f43591 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -577,7 +577,7 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
         // A call runs through its helpers (`call_op`), as the core loop's
         // arm runs it: a compiled callee is called directly, any other
         // activation switched to without a round trip through the loop.
-        if helped[k] || matches!(op, OpCode::Call | OpCode::CallKw) {
+        if helped[k] || matches!(op, OpCode::Call | OpCode::CallKw | OpCode::CallEx) {
             return Some(1);
         }
         if !native_at(code, k) {
@@ -814,6 +814,26 @@ mod stats {
         })
     }
 
+    /// [`super::run_pushed_directly`] declined, for `why`: `None`.
+    #[inline(always)]
+    pub(super) fn declined(why: &'static str) -> Option<u32> {
+        if enabled() {
+            note_declined(why);
+        }
+        None
+    }
+
+    #[cold]
+    #[inline(never)]
+    fn note_declined(why: &'static str) {
+        let mut counts = COUNTS.lock().unwrap_or_else(|e| e.into_inner());
+        let e = counts
+            .get_or_insert_with(HashMap::new)
+            .entry(("<pushed call declined>".to_owned(), why.to_owned()))
+            .or_default();
+        e[0] += 1;
+    }
+
     #[cold]
     pub(super) fn note(
         name: &str,
@@ -4325,7 +4345,7 @@ unsafe fn run_pushed_directly(
     unsafe {
         let sw = &mut *st.sw;
         if sw.pending.is_some() || std::ptr::eq(sw.cur, st.frame) {
-            return None;
+            return stats::declined("pending or in place");
         }
         let inl = &mut *sw.inl;
         let depth = inl.len();
@@ -4337,23 +4357,23 @@ unsafe fn run_pushed_directly(
             || top.direct
             || !std::ptr::eq(&raw const *top.frame, callee)
         {
-            return None;
+            return stats::declined("shape");
         }
         let interp = &*st.interp;
         let nested = interp.direct_calls.get();
         if nested >= DIRECT_CALL_DEPTH || crate::hot_gates::loop_gen() != st.snap_gen {
-            return None;
+            return stats::declined("depth or generation");
         }
         let cframe = &*callee;
         let ext = crate::code_vm_ext(&cframe.code)?;
         let nlocals = (*cframe.locals.as_ptr()).len();
         let Some(native) = ext.frame_jit.get(nlocals) else {
             ext.frame_jit.warm_switched(&cframe.code, ext, nlocals);
-            return None;
+            return stats::declined("no body");
         };
         let from = cframe.pc as usize;
         if from >= ext.dispatch_len || !native.enters_at(from) {
-            return None;
+            return stats::declined("no entry");
         }
         top.direct = true;
         Some(direct_run::<true>(
@@ -5105,6 +5125,51 @@ unsafe extern "C" fn h_call_kw(st: *mut State, code: *const CodeObject, pc: u64,
     }
 }
 
+/// `CALL_FUNCTION_EX` (the `pc`th instruction of `code`) on a stack `len`
+/// deep, as the core loop's arm runs it (`Interpreter::core_call_ex`, else
+/// its lane), the frame synced first: `0` the result in place (a compiled
+/// callee run from here, as for `CALL`, or a call finished in place),
+/// `RELOAD` the core loop goes on.
+unsafe extern "C" fn h_call_ex(st: *mut State, code: *const CodeObject, pc: u64, len: u64) -> u32 {
+    // SAFETY: the code passes its live state, its own code, one of its
+    // `CALL_FUNCTION_EX`s and its stack depth.
+    unsafe {
+        let _ = code;
+        let st = &mut *st;
+        let (pc, len) = (pc as usize, len as usize);
+        if st.sw.is_null() || len < 4 {
+            return CALL_DECLINED;
+        }
+        let start = len - 4;
+        let frame = &mut *st.frame;
+        frame.stack.set_len(len);
+        frame.pc = pc as u32;
+        let sw = &mut *st.sw;
+        *sw.last = st.last;
+        let saved_last = sw.last;
+        let interp = &mut *st.interp.cast_mut();
+        if !interp.core_call_ex(sw, pc, st.snap_gen) {
+            if sw.pending.is_none() && !interp.core_call_lane(sw) {
+                sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+            }
+            return if finished_in_place(st, start) { 0 } else { RELOAD };
+        }
+        if sw.pending.is_none() && !std::ptr::eq(sw.cur, st.frame) {
+            if let Some(r) = run_pushed_directly(st, pc, start, saved_last) {
+                return r;
+            }
+            if run_callee_directly(st, start) {
+                st.len = start + 1;
+                return 0;
+            }
+        }
+        if finished_in_place(st, start) {
+            return 0;
+        }
+        RELOAD
+    }
+}
+
 /// `RETURN_VALUE` (the `pc`th instruction) of an inline activation with
 /// the value atop a `len`-deep stack: the frame synced and the caller
 /// switched back to as the core loop's arm does (`RELOAD`), the state
@@ -7237,6 +7302,7 @@ impl<'a> Lower<'a> {
             // helper; a generator body's return isn't a switch.)
             OpCode::Call => return self.call_op(pc, ins.arg as usize),
             OpCode::CallKw => return self.call_kw(pc),
+            OpCode::CallEx => return self.call_ex(pc),
             // `await`'s and `yield from`'s delegate, through `h_await_iter`.
             OpCode::GetAwaitable | OpCode::GetYieldFromIter if self.depth > 0 => {
                 let s = self.top_to_mem();
@@ -9631,6 +9697,17 @@ impl<'a> Lower<'a> {
     /// `CALL_KW` through [`h_call_kw`] (as [`Self::call_op`]; the depth
     /// after is the static one).
     fn call_kw(&mut self, pc: usize) -> bool {
+        self.call_with(pc, h_call_kw as *const () as usize)
+    }
+
+    /// `CALL_FUNCTION_EX` through [`h_call_ex`] (as [`Self::call_kw`]).
+    fn call_ex(&mut self, pc: usize) -> bool {
+        self.call_with(pc, h_call_ex as *const () as usize)
+    }
+
+    /// A call instruction through `helper` (`h_call_kw`'s signature): the
+    /// result in place when it ran, anything else the core loop's.
+    fn call_with(&mut self, pc: usize, helper: usize) -> bool {
         let after = self.depths.get(pc + 1).copied().unwrap_or(-1);
         if after < 0 || self.depth == 0 {
             self.exit(INTERP, pc);
@@ -9642,11 +9719,7 @@ impl<'a> Lower<'a> {
         let pcv = self.b.ins().iconst(types::I64, pc as i64);
         let len = self.b.ins().iconst(types::I64, self.depth as i64);
         let r = self
-            .call(
-                h_call_kw as *const () as usize,
-                &[self.st, code, pcv, len],
-                true,
-            )
+            .call(helper, &[self.st, code, pcv, len], true)
             .expect("returns");
         let out = self.exit_with(pc, &[], INTERP);
         let ran = self.b.create_block();
diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index 81cf3795..6dfd8e8d 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -684,7 +684,7 @@ impl Interpreter {
                 OpCode::JumpForward => pc += 1 + ins.arg as usize,
                 OpCode::JumpBackward => {
                     // The back edge is the eval-breaker (as the core loop's).
-                    if self.countdown_out(1) || crate::hot_gates::loop_gen() != snap_gen {
+                    if self.gil_countdown <= 1 || crate::hot_gates::loop_gen() != snap_gen {
                         break None;
                     }
                     self.gil_countdown -= 1;
diff --git a/tests/regrtest/test_direct_call_shapes.py b/tests/regrtest/test_direct_call_shapes.py
index 9a9ab19d..6f67a0a6 100644
--- a/tests/regrtest/test_direct_call_shapes.py
+++ b/tests/regrtest/test_direct_call_shapes.py
@@ -71,6 +71,37 @@ def kw_calls():
     assert t == sum(2 * i for i in range(N)) + 123 * N, t
 
 
+class Point:
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+
+
+def spread_target(a, b, c=0):
+    if c == -1:
+        raise KeyError(a)
+    return a + b + c
+
+
+def spread_calls():
+    t = 0
+    for i in range(N):
+        args = (i, 1)
+        t += spread_target(*args)
+        t += spread_target(*args, **{"c": 2})
+        t += max(*[i, 3, 1])
+        p = Point(*args)
+        t += p.x + p.y
+    expect = sum((i + 1) + (i + 3) + max(i, 3) + (i + 1) for i in range(N))
+    assert t == expect, (t, expect)
+    try:
+        spread_target(*(5, 1), c=-1)
+    except KeyError as e:
+        assert e.args == (5,)
+    else:
+        raise AssertionError("no raise")
+
+
 class Dying:
     log = []
 
@@ -178,6 +209,7 @@ def spin(i):
 
 closures()
 kw_calls()
+spread_calls()
 del_timing()
 frames_through_closures()
 raising_closure()

From e58c3c3e0e220eade27a3209214160249e8f1d78 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 05:55:35 -0700
Subject: [PATCH 039/173] perf: keep method calls of calling effect leaves off
 the in-place helper

A compiled `LOAD_FAST x; x.m(...)` site whose method was a leaf when the
code compiled calls `h_local_method`, which evaluates a leaf callee in
place. An effect leaf that itself calls (go's `Square.find`, which
recurses and stores) is never evaluated in place: the helper probed the
site's caches, declined, and the call then went the ordinary way, about
270 wasted instructions per call. `method_site_in_place` now applies the
helper's own test, so such sites load the method and call it directly.
Simple direct-call gate trims ride along: the free-threading check (native
code only runs in a burst, which free-threaded mode never starts), the
greenlet probe's call, and the burst-stats call count's call.

Suite geomean 0.9933x the previous change (go 0.925, tomllib_loads
0.981, chaos 0.985); 0.9710x the base.
---
 crates/weavepy-vm/src/frame_jit.rs            |  3 ++-
 crates/weavepy-vm/src/lib.rs                  | 26 ++++++++++++++++---
 .../weavepy-vm/src/stdlib/greenlet_native.rs  | 10 ++++---
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index c1f43591..40826757 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -4289,11 +4289,12 @@ unsafe fn direct_call(
         let interp = &mut *st.interp.cast_mut();
         let nested = interp.direct_calls.get();
         let code: &CodeObject = code_rc;
+        // (Native code runs only in a burst, which free-threaded mode
+        // never starts.)
         if nested >= DIRECT_CALL_DEPTH
             || crate::hot_gates::loop_gen() != st.snap_gen
             || !native.enters_at(0)
             || !interp.inline_calls_ok()
-            || crate::gil::free_threading_enabled()
             || !(crate::tier2::jit_off_for_process()
                 || code.jit_hint.is_not_jitable()
                 || !code.jit_hint.is_compiled())
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index f1270e99..a164a166 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -65992,8 +65992,17 @@ mod burst_stats {
     static CALLS: [AtomicU64; 8] = [const { AtomicU64::new(0) }; 8];
 
     /// Count one call taking path `i` (one of the `CALL_*` constants).
-    #[inline]
+    /// (Every call path counts: off, it's one load in line.)
+    #[inline(always)]
     pub(crate) fn note_call(i: usize) {
+        if ENABLED.load(Ordering::Relaxed) != 0 {
+            note_call_slow(i);
+        }
+    }
+
+    #[cold]
+    #[inline(never)]
+    fn note_call_slow(i: usize) {
         if enabled() {
             CALLS[i].fetch_add(1, Ordering::Relaxed);
         }
@@ -71912,13 +71921,19 @@ pub(crate) fn call_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
 /// without an activation of its own.
 #[cfg(feature = "jit")]
 pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
-    // SAFETY: GIL-serialized; the borrows end before any refill.
+    // (An effect leaf that calls runs as an ordinary call, not in place:
+    // `core_local_method` declines it, so a native method-call helper
+    // there would only cost every call its probes.)
+    // SAFETY: GIL-serialized; the borrows end before any refill; a live
+    // site's code is its live function's.
     let leaf = ext
         .leaf_sites
         .get()
         .and_then(|s| s.get(pc))
         .and_then(|s| unsafe { &*s.0.get() }.as_ref())
-        .is_some_and(|site| site.func.strong_count() > 0);
+        .is_some_and(|site| {
+            site.func.strong_count() > 0 && !(site.effect && code_calls(unsafe { &*site.code }))
+        });
     leaf || ext
         .method_slots
         .get()
@@ -71927,7 +71942,10 @@ pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
             MethodSlotFn::Builtin(_) | MethodSlotFn::Leaf(..) => true,
             MethodSlotFn::Py(w) | MethodSlotFn::Static(w) => {
                 // SAFETY: the function is alive while the upgrade holds it.
-                w.upgrade().is_some_and(|f| fn_is_leaf(&f))
+                w.upgrade().is_some_and(|f| {
+                    fn_is_pure_leaf(&f)
+                        || (fn_is_leaf(&f) && !code_calls(unsafe { &*f.code.as_ptr() }))
+                })
             }
             _ => false,
         })
diff --git a/crates/weavepy-vm/src/stdlib/greenlet_native.rs b/crates/weavepy-vm/src/stdlib/greenlet_native.rs
index 5551ae75..a6e2e996 100644
--- a/crates/weavepy-vm/src/stdlib/greenlet_native.rs
+++ b/crates/weavepy-vm/src/stdlib/greenlet_native.rs
@@ -280,12 +280,16 @@ fn chain_contains(id: u64) -> bool {
 /// bounds, which are meaningless while running on a greenlet's own
 /// mmap'd stack, so those probes must fall back to their counted
 /// budgets here.
+#[inline]
 pub fn on_greenlet_stack() -> bool {
     // Nothing has ever created a greenlet: skip the thread-local reads
     // (this is polled on every lean Python call).
-    if !GREENLETS_EVER.load(Ordering::Relaxed) {
-        return false;
-    }
+    GREENLETS_EVER.load(Ordering::Relaxed) && on_greenlet_stack_slow()
+}
+
+#[cold]
+#[inline(never)]
+fn on_greenlet_stack_slow() -> bool {
     let main = MAIN_ID.with(|m| m.get());
     main != 0 && CURRENT.with(|c| c.get()) != main
 }

From 6e9534a1aa5e882fa2c5681d1165eca244b3ff4e Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 03:41:49 -0700
Subject: [PATCH 040/173] perf: read a live frame's line from its shell instead
 of syncing it

A frame object of a live activation now links to the activation's spine
shell (PyFrame::live) and reads lasti from it while the activation runs.
The dispatch loops already keep the shell's mirror current wherever code
can look at the frame (calls, out-of-line attribute loads through the
pending list, slow steps, hand-backs to the general loop), so
materializing a frame no longer bumps the loop generation, and a held
frame object (a sys._getframe handle, a traceback while its handler
runs) no longer pins its activation to a per-instruction sync on the
general path. The link is set when a shell takes the object and cleared
when the shell leaves the spine (copying the final lasti) or dies.

- The core loop's in-line frame attribute read declines f_lineno of a
  live frame; core_attr_lane serves frame and code fields after putting
  the activation on the pending list, which makes its lasti current.
- sys._getframe and sys._getframemodulename run in the core loop's
  builtin lane (they make the spine whole themselves) and read only the
  frame stack handle.
- A frameless tier-2 activation whose frame object a call made (its own
  sys._getframe()) leaves native code after that call; the interpreter
  continuation adopts the object and its locals, so the object keeps
  tracking the activation (it used to stay at the call's line).
- While the globals-__missing__ gate is up, every 1024th builtin lookup
  has the next safe point prune dead owners: importing traceback (via
  _colorize's dataclasses) left the gate up for good in programs that
  run no finalizers, sending every builtin LOAD_GLOBAL to the helper.

Also fixes stale f_lineno reads that the old protocol missed: a frame
reading its own line in a loop tier 2 or the frame JIT compiled, and a
raising callee's frame in a traceback under the hot paths.

Measured (instructions, pinned seed): sys._getframe(1).f_code.co_name
from a call chain 11.3k -> 8.3k per call; logging findCaller 33.3k ->
16.7k; one log.info 171k -> 151k. Suite geomean 1.569x -> 1.561x CPython
(logging_mix -7.2%, raytrace -1.8%, chaos -2.1%, nothing worse than
+0.4%).
---
 crates/weavepy-vm/src/hot_gates.rs            |   9 +-
 crates/weavepy-vm/src/lib.rs                  | 311 ++++++++----------
 crates/weavepy-vm/src/object.rs               | 125 ++++---
 crates/weavepy-vm/src/stdlib/sys.rs           |  12 +-
 .../weavepy-vm/src/stdlib/weave_frame_mod.rs  |   1 +
 crates/weavepy-vm/src/tier2.rs                |  52 ++-
 crates/weavepy-vm/src/vm_singletons.rs        |   6 +
 tests/regrtest/test_frame_live_lineno.py      | 175 ++++++++++
 8 files changed, 465 insertions(+), 226 deletions(-)
 create mode 100644 tests/regrtest/test_frame_live_lineno.py

diff --git a/crates/weavepy-vm/src/hot_gates.rs b/crates/weavepy-vm/src/hot_gates.rs
index d84b931b..7be3098a 100644
--- a/crates/weavepy-vm/src/hot_gates.rs
+++ b/crates/weavepy-vm/src/hot_gates.rs
@@ -64,13 +64,12 @@ static HOT: AtomicU32 = AtomicU32::new(0);
 
 /// RFC 0065 (WS1): the dispatch loop's *generation* word. Bumped by
 /// every mutation that can change the loop prologue's decisions —
-/// hot-gate bits ([`set`]/[`clear`] below), observer registration
-/// (`trace::bump_observer_gen`), and frame materialization
-/// (`FrameShell::materialize`). The dispatch loop caches a snapshot
+/// hot-gate bits ([`set`]/[`clear`] below) and observer registration
+/// (`trace::bump_observer_gen`). The dispatch loop caches a snapshot
 /// of the prologue's inputs keyed by this generation: while the
 /// generation is unchanged *and* the snapshot says "quiet" (no
-/// pending work, no observers, no materialized frame), the loop runs one relaxed load + compare per
-/// instruction instead of the full ten-plus-probe prologue.
+/// pending work, no observers), the loop runs one relaxed load + compare
+/// per instruction instead of the full ten-plus-probe prologue.
 ///
 /// Same producer/consumer discipline as [`HOT`]: a spurious bump is
 /// always safe (one cold re-snapshot); a missed bump is never
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index a164a166..0b2064b8 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -905,6 +905,9 @@ pub struct Interpreter {
     /// Finalizer drains left before the next prune of
     /// `globals_missing_hooks` while `globals_missing_any` is set.
     globals_missing_prune_in: Cell<u32>,
+    /// Builtin lookups that asked for a `__missing__` owner since the
+    /// last forced prune (see `globals_missing_owner`).
+    globals_missing_asks: Cell<u32>,
     /// Non-zero while the VM is lazily loading one of its own machinery
     /// modules (e.g. `importlib._bootstrap` for `module.__repr__`). In
     /// CPython those are frozen and fully initialized before user code
@@ -985,11 +988,6 @@ pub struct Interpreter {
     /// exact per-instruction event stream. Refreshed from the dispatch
     /// loop's [`crate::trace::ObserverSnapshot`] every iteration.
     fuse_off: bool,
-    /// A handled exception was just released (`POP_EXCEPT`): the running
-    /// frame's object may have lost its last outside holder (the
-    /// traceback), so the dispatch loop re-probes it on the next
-    /// instruction instead of waiting for its stride.
-    recheck_frame_observed: bool,
     /// `WEAVEPY_NO_QUIET`: pin every dispatch-loop iteration to the full
     /// prologue (RFC 0065 bisection aid). Read once at construction so a
     /// frame entry does not pay a `OnceLock` probe for it (RFC 0077 WS3).
@@ -1238,6 +1236,7 @@ impl Default for Interpreter {
             globals_missing_hooks: RefCell::new(Vec::new()),
             globals_missing_any: Cell::new(false),
             globals_missing_prune_in: Cell::new(0),
+            globals_missing_asks: Cell::new(0),
             internal_import_depth: 0,
             frame_locals_pool: ThreadCell::new(Vec::new()),
             inline_pool: Vec::new(),
@@ -1250,7 +1249,6 @@ impl Default for Interpreter {
             direct_calls: std::cell::Cell::new(0),
             sum_fold: None,
             fuse_off: false,
-            recheck_frame_observed: false,
             quiet_off: crate::hot_gates::env_flags::no_quiet(),
             burst_on: !crate::hot_gates::env_flags::no_burst()
                 && !crate::hot_gates::env_flags::no_quiet()
@@ -1368,6 +1366,7 @@ impl Interpreter {
             globals_missing_hooks: RefCell::new(Vec::new()),
             globals_missing_any: Cell::new(false),
             globals_missing_prune_in: Cell::new(0),
+            globals_missing_asks: Cell::new(0),
             internal_import_depth: 0,
             frame_locals_pool: ThreadCell::new(Vec::new()),
             inline_pool: Vec::new(),
@@ -1380,7 +1379,6 @@ impl Interpreter {
             direct_calls: std::cell::Cell::new(0),
             sum_fold: None,
             fuse_off: false,
-            recheck_frame_observed: false,
             quiet_off: crate::hot_gates::env_flags::no_quiet(),
             burst_on: !crate::hot_gates::env_flags::no_burst()
                 && !crate::hot_gates::env_flags::no_quiet()
@@ -4368,7 +4366,7 @@ impl Interpreter {
                     shell
                         .lasti
                         .store(pc as u32, std::sync::atomic::Ordering::Relaxed);
-                    py_frame.lasti.set(pc as u32);
+                    py_frame.set_lasti(pc as u32);
                 }
             }
             if let Err(e) = self.fire_call_event(&py_frame, mon_event, Object::None) {
@@ -4571,11 +4569,6 @@ impl Interpreter {
         let quiet_off = self.quiet_off;
         let mut loop_snap_gen: u64 = 0; // stale → first iteration derives
         let mut loop_quiet = false;
-        // The loops are off only because a frame object is observed (see
-        // `frame_object_observed`): re-derive once nothing outside the
-        // activation holds it any more (an `except` clause dropped the
-        // traceback).
-        let mut frame_blocks_quiet = false;
         let result = 'activation: loop {
             // RFC 0039 (WS2) / RFC 0059 (WS2): cooperative GIL hand-off.
             // A plain interpreter-local countdown — one register
@@ -4592,25 +4585,13 @@ impl Interpreter {
             // a prologue input changed. The observer snapshot refresh
             // rides the same generation (observer mutation bumps it).
             let lgen = crate::hot_gates::loop_gen();
-            // The frame object's holder (a live traceback, a
-            // `sys._getframe` handle) is re-probed on a stride, not per
-            // instruction: the probe reads a cell and a reference count,
-            // and an exception handler paid it for every instruction it
-            // ran. A few extra instructions on the full path after the
-            // holder goes away cost far less.
-            let rederive = frame_blocks_quiet
-                && (std::mem::take(&mut self.recheck_frame_observed)
-                    || self.gil_countdown.trailing_zeros() >= 4)
-                && !Self::frame_object_observed(&shell, py_frame_slot.as_ref(), frame);
-            if lgen != loop_snap_gen || rederive {
+            if lgen != loop_snap_gen {
                 loop_snap_gen = lgen;
                 obs.refresh();
                 self.fuse_off = obs.any;
-                let unobserved = !quiet_off && crate::hot_gates::load() == 0 && !obs.any;
-                let frame_observed =
-                    Self::frame_object_observed(&shell, py_frame_slot.as_ref(), frame);
-                frame_blocks_quiet = unobserved && frame_observed;
-                loop_quiet = unobserved && !frame_observed;
+                // A frame object of this activation doesn't keep the loops
+                // off: it reads `lasti` from the shell (see `PyFrame::live`).
+                loop_quiet = !quiet_off && crate::hot_gates::load() == 0 && !obs.any;
             }
             // The leaf burst: while the loop is quiet, run the straight
             // run of leaf instructions at `frame.pc` in one tight loop
@@ -4645,27 +4626,11 @@ impl Interpreter {
                 } else {
                     None
                 };
-                // A frame object only this activation holds skipped the
-                // per-instruction `lasti` sync: bring it current before the
-                // prologue (or a raise's traceback) reads it.
-                if quiet_exit.is_some()
-                    && shell
-                        .has_materialized
-                        .load(std::sync::atomic::Ordering::Relaxed)
-                {
-                    // The instruction that just ran (or raised), as the
-                    // per-instruction sync would have left it; after a
-                    // yield back to the prologue, the next one.
-                    let at = match &quiet_exit {
-                        Some(QuietExit::Outcome { cur_pc, .. }) => *cur_pc as u32,
-                        _ => frame.pc,
-                    };
-                    if let Some(py) = shell.materialized.borrow().as_ref() {
-                        py.lasti.set(at);
-                    }
-                } else if let Some(exit) = &quiet_exit {
-                    // Likewise the shell a lazy traceback entry will
-                    // materialize from (see `pop_frame_shell`).
+                // The shell a frame object reads its `lasti` from, or a lazy
+                // traceback entry will materialize from (see
+                // `pop_frame_shell`), at the instruction that just ran (or
+                // raised); a yield back to the prologue left it at the next.
+                if let Some(exit) = &quiet_exit {
                     Self::sync_lazy_tb_lasti(Some(&shell), &frame.code, exit);
                 }
                 match quiet_exit {
@@ -4683,37 +4648,10 @@ impl Interpreter {
                         shell
                             .lasti
                             .store(frame.pc, std::sync::atomic::Ordering::Relaxed);
-                        if shell
-                            .has_materialized
-                            .load(std::sync::atomic::Ordering::Relaxed)
-                        {
-                            if let Some(py) = shell.materialized.borrow().as_ref() {
-                                py.lasti.set(frame.pc);
-                            }
-                        }
                         instruction_ran = true;
                         self.step_hot(frame)
                     }
                 }
-            } else if frame_blocks_quiet {
-                // Quiet but for a frame object something outside holds (an
-                // exception's traceback, while its handler runs): nothing
-                // the full prologue serves is pending, so each instruction
-                // only brings the frame's `lasti` current before it runs.
-                cur_pc = frame.pc as usize;
-                shell
-                    .lasti
-                    .store(frame.pc, std::sync::atomic::Ordering::Relaxed);
-                if shell
-                    .has_materialized
-                    .load(std::sync::atomic::Ordering::Relaxed)
-                {
-                    self.ensure_top_py_frame(&mut py_frame_slot)
-                        .lasti
-                        .set(frame.pc);
-                }
-                instruction_ran = true;
-                self.step_hot(frame)
             } else {
                 // RFC 0059 (WS2): the unified eval-breaker word. One relaxed
                 // load answers "is *any* deferred work pending?" for the six
@@ -4805,7 +4743,7 @@ impl Interpreter {
                 self.fuse_off = obs.any;
                 if obs.any && !is_gen_bootstrap {
                     let py_frame = self.ensure_top_py_frame(&mut py_frame_slot);
-                    py_frame.lasti.set(frame.pc);
+                    py_frame.set_lasti(frame.pc);
                     let line = py_frame.current_lineno();
                     // RFC 0051 (WS4): CPython 3.13 line-event semantics.
                     // A `'line'` event fires when execution reaches an
@@ -5279,7 +5217,7 @@ impl Interpreter {
                                 // fires at the handler entry.
                                 if let Some(inst) = handled_arg {
                                     let py_frame = self.ensure_top_py_frame(&mut py_frame_slot);
-                                    py_frame.lasti.set(frame.pc);
+                                    py_frame.set_lasti(frame.pc);
                                     if let Err(mon_e) = self.fire_monitoring_event(
                                         &py_frame,
                                         crate::trace::EVENT_EXCEPTION_HANDLED,
@@ -5488,7 +5426,7 @@ impl Interpreter {
             }
             let materialized = frame.py_frame.clone();
             if let Some(existing) = &materialized {
-                existing.lasti.set(frame.pc);
+                existing.set_lasti(frame.pc);
                 existing.invalidate_locals();
                 existing.on_stack.set(existing.on_stack.get() + 1);
             }
@@ -5503,6 +5441,9 @@ impl Interpreter {
             cached
                 .has_materialized
                 .store(materialized.is_some(), std::sync::atomic::Ordering::Relaxed);
+            if let Some(py) = &materialized {
+                cached.link(py);
+            }
             *cached.materialized.borrow_mut() = materialized;
             let shell = cached.clone();
             self.frame_stack.borrow_mut().push(shell.clone());
@@ -5512,7 +5453,7 @@ impl Interpreter {
             frame.code.is_generator || frame.code.is_coroutine || frame.code.is_async_generator;
         let materialized = frame.py_frame.clone();
         if let Some(existing) = &materialized {
-            existing.lasti.set(frame.pc);
+            existing.set_lasti(frame.pc);
             existing.invalidate_locals();
             existing.on_stack.set(existing.on_stack.get() + 1);
         }
@@ -5565,6 +5506,9 @@ impl Interpreter {
                 *m.gen_owner.get_mut() = gen_owner;
                 m.lasti = std::sync::atomic::AtomicU32::new(frame.pc);
                 m.has_materialized = std::sync::atomic::AtomicBool::new(materialized.is_some());
+                if let Some(py) = &materialized {
+                    m.link(py);
+                }
                 *m.materialized.get_mut() = materialized;
                 self.frame_stack.borrow_mut().push(pooled.clone());
                 return pooled;
@@ -5586,6 +5530,9 @@ impl Interpreter {
             materialized: RefCell::new(materialized),
             tb_refs: std::sync::atomic::AtomicU32::new(0),
         });
+        if let Some(py) = shell.materialized.borrow().as_ref() {
+            shell.link(py);
+        }
         self.frame_stack.borrow_mut().push(shell.clone());
         shell
     }
@@ -5627,6 +5574,7 @@ impl Interpreter {
     fn push_materialized_frame(&self, py: &Rc<PyFrame>) {
         py.on_stack.set(py.on_stack.get() + 1);
         let shell = Rc::new(crate::object::FrameShell::from_py_frame(py));
+        shell.link(py);
         self.frame_stack.borrow_mut().push(shell);
     }
 
@@ -5669,40 +5617,6 @@ impl Interpreter {
 
     /// Dispatch-loop helper: materialise the current activation's
     /// `PyFrame` once and cache it in the loop-local slot.
-    /// Whether this activation's frame object (if one was materialized)
-    /// is reachable from anything but the activation's own bookkeeping —
-    /// the shell's cell, the loop's slot, the frame's link: a traceback
-    /// or a Python reference that may read `f_lineno` at any moment, so
-    /// the loop keeps its `lasti` current instruction by instruction.
-    /// Otherwise the quiet loops may run, syncing it whenever they hand
-    /// back and whenever the object is fetched again (see
-    /// `FrameShell::materialize`).
-    fn frame_object_observed(
-        shell: &crate::object::FrameShell,
-        slot: Option<&Rc<PyFrame>>,
-        frame: &Frame,
-    ) -> bool {
-        if !shell
-            .has_materialized
-            .load(std::sync::atomic::Ordering::Relaxed)
-        {
-            return false;
-        }
-        // Borrowed, not cloned: this runs per instruction while a
-        // traceback holds the frame object, and the clone's own count
-        // would be two more atomics each time.
-        let Ok(cell) = shell.materialized.try_borrow() else {
-            return true;
-        };
-        let Some(py) = cell.as_ref() else {
-            return false;
-        };
-        let own = 1
-            + usize::from(slot.is_some_and(|s| Rc::ptr_eq(s, py)))
-            + usize::from(frame.py_frame.as_ref().is_some_and(|p| Rc::ptr_eq(p, py)));
-        Rc::strong_count(py) > own
-    }
-
     fn ensure_top_py_frame(&self, slot: &mut Option<Rc<PyFrame>>) -> Rc<PyFrame> {
         if let Some(py) = slot {
             return py.clone();
@@ -5757,6 +5671,7 @@ impl Interpreter {
             extra_locals: RefCell::new(None),
             cleared: Cell::new(false),
             lazy_back: RefCell::new(None),
+            live: std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()),
         })
     }
 
@@ -5775,7 +5690,7 @@ impl Interpreter {
             self.exc_info_stack.borrow_mut().pop();
         }
         frame.pc = jump.target_pc;
-        py_frame.lasti.set(jump.target_pc);
+        py_frame.set_lasti(jump.target_pc);
         py_frame.override_lineno.set(None);
         // Force a fresh 'line' event at the landing site.
         py_frame.last_line.set(None);
@@ -5831,7 +5746,7 @@ impl Interpreter {
         }
         let suspended = event == TraceEvent::Yield;
         let (jump, resolved_line) =
-            crate::linejump::compute_jump(&fr.code, fr.lasti.get(), new_lineno, suspended)?;
+            crate::linejump::compute_jump(&fr.code, fr.lasti_now(), new_lineno, suspended)?;
         // CPython binds `None` to any still-unbound local at the target
         // (rather than crashing on a LOAD_FAST of a name the compiler
         // "proved" bound), warning first so an escalating filter can
@@ -6118,9 +6033,10 @@ impl Interpreter {
     }
 
     /// A quiet loop leaves its shell's `lasti` behind the instruction it
-    /// stopped on. A lazy traceback entry materializes from the shell
-    /// once the activation leaves the spine (see `pop_frame_shell`), so
-    /// bring it current for the frame object's final `f_lineno`.
+    /// stopped on. A frame object reads it from there (see
+    /// `PyFrame::live`), and a lazy traceback entry materializes from the
+    /// shell once the activation leaves the spine (see `pop_frame_shell`),
+    /// so bring it current for the frame object's final `f_lineno`.
     /// A `RERAISE` leaves the restored raise site there itself (PEP 626).
     #[inline]
     fn sync_lazy_tb_lasti(
@@ -6129,12 +6045,11 @@ impl Interpreter {
         exit: &QuietExit,
     ) {
         if let (Some(shell), QuietExit::Outcome { cur_pc, stepped }) = (shell, exit) {
-            if shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) != 0
-                && !(stepped.is_err()
-                    && code
-                        .instructions
-                        .get(*cur_pc)
-                        .is_some_and(|i| i.op == OpCode::Reraise))
+            if !(stepped.is_err()
+                && code
+                    .instructions
+                    .get(*cur_pc)
+                    .is_some_and(|i| i.op == OpCode::Reraise))
             {
                 shell
                     .lasti
@@ -6165,11 +6080,8 @@ impl Interpreter {
         let existing = caller.materialized.borrow().clone();
         match existing {
             Some(back) => {
-                // A new holder of the caller's frame object: its loop
-                // re-derives whether the object is observed.
                 caller.refresh_materialized(&back);
                 py.set_back(Some(back));
-                crate::hot_gates::bump_loop_gen();
             }
             None => {
                 *py.lazy_back.borrow_mut() = Some(crate::object::LazyShellRef::new(caller));
@@ -6207,6 +6119,7 @@ impl Interpreter {
                 return popped;
             }
             if let Some(py) = shell.materialized.borrow().as_ref() {
+                py.unlink_live(shell);
                 py.on_stack.set(py.on_stack.get().saturating_sub(1));
                 // A generator-family frame that just suspended (yielded)
                 // or finished is no longer reachable from a live caller.
@@ -6762,7 +6675,7 @@ impl Interpreter {
             let tb = PyTraceback::new(
                 py_frame.clone(),
                 py_frame.last_line.get().unwrap_or(1),
-                py_frame.lasti.get(),
+                py_frame.lasti_now(),
                 None,
             );
             *tb.next.borrow_mut() = inherited_tb.clone();
@@ -7033,7 +6946,7 @@ impl Interpreter {
         if crate::trace::monitoring_union_mask_cached() & crate::trace::event_mask(event_idx) == 0 {
             return Ok(());
         }
-        let pc = py_frame.lasti.get();
+        let pc = py_frame.lasti_now();
         let code = py_frame.code.clone();
         let code_obj = Object::Code(code.clone());
         let offset = Object::Int(i64::from(code.cpython_lasti(pc)));
@@ -15015,8 +14928,8 @@ impl Interpreter {
     /// body must be reached through [`Self::call`]'s interpreter-aware
     /// dispatch. A method must be bound to its exact builtin receiver; a
     /// function must be one of the common builtins that never look at the
-    /// calling frame (`locals()`, `sys._getframe()` and the like keep the
-    /// full handler, which makes the frame stack whole first).
+    /// calling frame, or make the frame stack whole themselves (`locals()`
+    /// and the like keep the full handler, which makes it whole first).
     fn builtin_lane(
         &self,
         b: &Rc<crate::object::BuiltinFn>,
@@ -17432,7 +17345,6 @@ impl Interpreter {
                 let top = self.exc_info_stack.borrow_mut().pop();
                 // See the full handler: the handled exception dies here
                 // unless something else holds it.
-                self.recheck_frame_observed = true;
                 if let Some(pe) = top {
                     self.release(pe.instance);
                 }
@@ -21335,28 +21247,7 @@ impl Interpreter {
         let recv = recv?;
         let name = code.names.get(name_idx as usize)?.as_str();
         match recv {
-            // A code object's fields and a frame's plain ones (neither type
-            // can be subclassed, and no getter here runs Python); the full
-            // path keeps the deprecated `co_lnotab`, a generator frame's
-            // re-derived `f_back`, and a line-0 `f_lineno`.
-            Object::Code(c) if name != "co_lnotab" => crate::builtins::code_synthetic_attr(c, name),
-            Object::Frame(fr) => match name {
-                "f_code" => Some(Object::Code(fr.code.clone())),
-                "f_globals" => Some(Object::Dict(fr.globals.clone())),
-                "f_builtins" => Some(Object::Dict(fr.builtins.clone())),
-                "f_lineno" => match fr.current_lineno() {
-                    0 => None,
-                    line => Some(Object::Int(i64::from(line))),
-                },
-                "f_back"
-                    if !(fr.code.is_generator
-                        || fr.code.is_coroutine
-                        || fr.code.is_async_generator) =>
-                {
-                    Some(fr.back_frame().map_or(Object::None, Object::Frame))
-                }
-                _ => None,
-            },
+            Object::Code(_) | Object::Frame(_) => Self::frame_plain_attr(recv, name, false),
             // `x.__class__` of a builtin value: its exact type (no instance
             // or class can override it there).
             Object::Generator(_)
@@ -21377,6 +21268,49 @@ impl Interpreter {
             {
                 Some(Object::Type(crate::builtins::class_of(recv)))
             }
+            _ => self.leaf_plain_attr_value_rest(recv, name),
+        }
+    }
+
+    /// A code object's fields and a frame's plain ones (neither type can
+    /// be subclassed, and no getter here runs Python); the full path keeps
+    /// the deprecated `co_lnotab`, a generator frame's re-derived
+    /// `f_back`, and a line-0 `f_lineno`. A live frame's line needs its
+    /// activation's `lasti` current (see `PyFrame::live`), which the
+    /// running one's is only once it waits on the pending list: `synced`
+    /// says it does.
+    #[inline]
+    fn frame_plain_attr(recv: &Object, name: &str, synced: bool) -> Option<Object> {
+        match recv {
+            Object::Code(c) if name != "co_lnotab" => crate::builtins::code_synthetic_attr(c, name),
+            Object::Frame(fr) => match name {
+                "f_code" => Some(Object::Code(fr.code.clone())),
+                "f_globals" => Some(Object::Dict(fr.globals.clone())),
+                "f_builtins" => Some(Object::Dict(fr.builtins.clone())),
+                "f_lineno"
+                    if synced || fr.live.load(std::sync::atomic::Ordering::Relaxed).is_null() =>
+                {
+                    match fr.current_lineno() {
+                        0 => None,
+                        line => Some(Object::Int(i64::from(line))),
+                    }
+                }
+                "f_back"
+                    if !(fr.code.is_generator
+                        || fr.code.is_coroutine
+                        || fr.code.is_async_generator) =>
+                {
+                    Some(fr.back_frame().map_or(Object::None, Object::Frame))
+                }
+                _ => None,
+            },
+            _ => None,
+        }
+    }
+
+    /// The rest of [`Self::leaf_plain_attr_value`]'s shapes.
+    fn leaf_plain_attr_value_rest(&self, recv: &Object, name: &str) -> Option<Object> {
+        match recv {
             Object::List(_)
             | Object::Dict(_)
             | Object::Set(_)
@@ -24384,7 +24318,6 @@ impl Interpreter {
                     let top = self.exc_info_stack.borrow_mut().pop();
                     // See the full handler: the handled exception dies
                     // here unless something else holds it.
-                    self.recheck_frame_observed = true;
                     last = pc;
                     pc += 1;
                     if let Some(pe) = top {
@@ -30549,7 +30482,6 @@ impl Interpreter {
                 // `except` block (the implicit `del` of the bound name plus
                 // the per-frame exc-state pop): it dies here unless something
                 // else holds it.
-                self.recheck_frame_observed = true;
                 if let Some(pe) = popped {
                     self.release(pe.instance);
                 }
@@ -30624,7 +30556,7 @@ impl Interpreter {
                             .lasti
                             .store(orig, std::sync::atomic::Ordering::Relaxed);
                         if let Some(py_frame) = shell.materialized.borrow().as_ref() {
-                            py_frame.lasti.set(orig);
+                            py_frame.set_lasti(orig);
                         }
                     }
                 }
@@ -31536,12 +31468,10 @@ impl Interpreter {
         frame: &Frame,
         cur_pc: usize,
     ) {
-        if shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) != 0 {
-            // See `sync_lazy_tb_lasti`.
-            shell
-                .lasti
-                .store(cur_pc as u32, std::sync::atomic::Ordering::Relaxed);
-        }
+        // See `sync_lazy_tb_lasti`.
+        shell
+            .lasti
+            .store(cur_pc as u32, std::sync::atomic::Ordering::Relaxed);
         self.pop_frame_shell();
         Self::settle_exited_frame_object(&shell, frame, true);
         self.recycle_frame_shell(shell);
@@ -31723,7 +31653,7 @@ impl Interpreter {
                                 // Cache it on generator-family frames so
                                 // `gi_frame` keeps a stable identity.
                                 let py = self.build_py_frame(frame, None);
-                                py.lasti.set(lasti);
+                                py.set_lasti(lasti);
                                 if frame.code.is_generator
                                     || frame.code.is_coroutine
                                     || frame.code.is_async_generator
@@ -31793,7 +31723,7 @@ impl Interpreter {
         let new_tb = Rc::new(PyTraceback::new(
             py_frame.clone(),
             py_frame.last_line.get().unwrap_or(1),
-            py_frame.lasti.get(),
+            py_frame.lasti_now(),
             None,
         ));
         inst.slot_set("__traceback__", Object::Traceback(new_tb));
@@ -32317,6 +32247,18 @@ impl Interpreter {
     /// The dict-subclass instance registered (by `exec`/`eval`) as the
     /// owner of `globals`, if any.
     fn globals_missing_owner(&self, globals: &Rc<RefCell<DictData>>) -> Option<Object> {
+        // While the gate is up every builtin `LOAD_GLOBAL` asks here, and
+        // a program that runs no finalizers may never drain them to notice
+        // the last owner die: every so often, have the next safe point
+        // drain and prune (see `prune_globals_missing_hooks_sometimes`).
+        let n = self.globals_missing_asks.get();
+        if n >= 1024 {
+            self.globals_missing_asks.set(0);
+            self.globals_missing_prune_in.set(0);
+            gc_trace::mark_maybe_dead();
+        } else {
+            self.globals_missing_asks.set(n + 1);
+        }
         let hooks = self.globals_missing_hooks.borrow();
         if hooks.is_empty() {
             return None;
@@ -33325,7 +33267,7 @@ impl Interpreter {
                     // lineno_matches_lasti at the exec 'call' event).
                     let line = fr.current_lineno();
                     if line == 0 {
-                        let at_module_resume = fr.lasti.get() == 0
+                        let at_module_resume = fr.lasti_now() == 0
                             && fr.code.name == "<module>"
                             && fr.code.instructions.first().map(|x| x.op)
                                 == Some(weavepy_compiler::bytecode::OpCode::Resume);
@@ -33345,7 +33287,7 @@ impl Interpreter {
                     }
                 }
                 "f_lasti" => Ok(Object::Int(i64::from(
-                    fr.code.cpython_lasti(fr.lasti.get()),
+                    fr.code.cpython_lasti(fr.lasti_now()),
                 ))),
                 "f_back" => {
                     // A generator-family frame drops its resumer link when
@@ -42722,7 +42664,7 @@ impl Interpreter {
                 frame.py_frame = Some(py.clone());
                 py
             });
-            py.lasti.set(frame.pc);
+            py.set_lasti(frame.pc);
             let line = frame
                 .code
                 .linetable
@@ -42972,7 +42914,7 @@ impl Interpreter {
                         let tb = Rc::new(PyTraceback::new(
                             pf.clone(),
                             pf.last_line.get().unwrap_or(1),
-                            pf.lasti.get(),
+                            pf.lasti_now(),
                             None,
                         ));
                         inst.slot_set("__traceback__", Object::Traceback(tb));
@@ -43099,7 +43041,7 @@ impl Interpreter {
             frame.py_frame = Some(py.clone());
             py
         });
-        py.lasti.set(frame.pc);
+        py.set_lasti(frame.pc);
         py.set_back(self.materialize_top_py_frame());
         self.push_materialized_frame(&py);
         let hook_result =
@@ -46166,6 +46108,24 @@ impl Interpreter {
         let ins = frame.code.instructions[pc];
         frame.pc = pc as u32 + 1;
         let pending = self.core_pending_enter(sw, frame, pc);
+        // A frame's or code object's plain field, now that this
+        // activation's `lasti` is current for its own frame's line.
+        if ins.op == OpCode::LoadAttr
+            && matches!(frame.stack.last(), Some(Object::Frame(_) | Object::Code(_)))
+        {
+            let v = match (frame.stack.last(), frame.code.names.get(ins.arg as usize)) {
+                (Some(recv), Some(name)) => Self::frame_plain_attr(recv, name, true),
+                _ => None,
+            };
+            if let Some(v) = v {
+                if let Some(top) = frame.stack.last_mut() {
+                    let recv = std::mem::replace(top, v);
+                    self.release(recv);
+                }
+                self.core_pending_exit(sw, pending);
+                return Ok(());
+            }
+        }
         let r = match ins.op {
             OpCode::LoadAttr => match self.load_attr_certain_miss(frame, ins.arg) {
                 Some(err) => Err(err),
@@ -71076,7 +71036,10 @@ fn builtin_method_via_call(name: &str) -> bool {
 /// Builtin functions the core loop may call in place (see
 /// `Interpreter::builtin_lane`): common ones that never look at the
 /// calling frame, so the frame stack needn't be whole while they run.
-/// (Python code they call makes it whole as it needs to.)
+/// (Python code they call makes it whole as it needs to.) The `sys`
+/// frame readers make it whole themselves (`sync_frame_spine`), the
+/// calling activation waiting on the pending list with its `lasti` at the
+/// call.
 fn builtin_fn_lane_ok(name: &str) -> bool {
     matches!(
         name,
@@ -71104,6 +71067,8 @@ fn builtin_fn_lane_ok(name: &str) -> bool {
             | "monotonic_ns" | "heappush" | "heappop" | "heapify" | "heapreplace"
             | "heappushpop" | "bisect" | "bisect_left" | "bisect_right" | "insort"
             | "insort_left" | "insort_right"
+            // sys
+            | "_getframe" | "_getframemodulename"
     )
 }
 
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 5080f27d..2e7b44e8 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -458,10 +458,20 @@ pub struct PyFrame {
     pub code: Rc<CodeObject>,
     pub globals: Rc<RefCell<DictData>>,
     pub builtins: Rc<RefCell<DictData>>,
-    /// Index of the current instruction inside [`Self::code`]. Updated
-    /// per-instruction by the dispatch loop while this frame is the
-    /// active one.
+    /// Index of the current instruction inside [`Self::code`]. While the
+    /// activation is live its shell's mirror is the current one (see
+    /// [`Self::live`]): read through [`Self::lasti_now`] and write through
+    /// [`Self::set_lasti`].
     pub lasti: Cell<u32>,
+    /// The spine shell of this frame's activation while it is live, null
+    /// otherwise. The dispatch loops keep the shell's `lasti` current at
+    /// every point where code may look at the frame (each call, each
+    /// out-of-line attribute load, each hand-back to the general loop),
+    /// so a frame object reads it from there instead of making its
+    /// activation sync the object instruction by instruction. Set when
+    /// the shell takes the object, cleared when the shell leaves the
+    /// spine or dies.
+    pub(crate) live: std::sync::atomic::AtomicPtr<FrameShell>,
     /// The enclosing frame (the next-outer in the call stack), `None`
     /// for the module frame.
     pub back: RefCell<Option<Rc<PyFrame>>>,
@@ -639,10 +649,52 @@ impl PyFrame {
         if let Some(v) = self.override_lineno.get() {
             return v;
         }
-        let pc = self.lasti.get() as usize;
+        let pc = self.lasti_now() as usize;
         self.code.linetable.get(pc).copied().unwrap_or(0)
     }
 
+    /// The current instruction index: the live shell's mirror while the
+    /// activation runs (see [`Self::live`]), unless a trace function's
+    /// line jump or `f_lineno` override owns the value.
+    pub fn lasti_now(&self) -> u32 {
+        let live = self.live.load(std::sync::atomic::Ordering::Relaxed);
+        if !live.is_null()
+            && self.override_lineno.get().is_none()
+            && self.pending_jump.try_borrow().is_ok_and(|j| j.is_none())
+        {
+            // SAFETY: a linked shell is alive: the link is cleared when the
+            // shell leaves its spine and again when it is dropped.
+            let at = unsafe { (*live).lasti.load(std::sync::atomic::Ordering::Relaxed) };
+            self.lasti.set(at);
+            return at;
+        }
+        self.lasti.get()
+    }
+
+    /// Set the current instruction index, in the live shell's mirror too.
+    pub fn set_lasti(&self, at: u32) {
+        self.lasti.set(at);
+        let live = self.live.load(std::sync::atomic::Ordering::Relaxed);
+        if !live.is_null() {
+            // SAFETY: as in `lasti_now`.
+            unsafe {
+                (*live)
+                    .lasti
+                    .store(at, std::sync::atomic::Ordering::Relaxed)
+            };
+        }
+    }
+
+    /// Settle the frame as its activation leaves the spine on `shell`:
+    /// the shell's last instruction becomes the object's own.
+    pub(crate) fn unlink_live(&self, shell: &FrameShell) {
+        if std::ptr::eq(self.live.load(std::sync::atomic::Ordering::Relaxed), shell) {
+            shell.refresh_materialized(self);
+            self.live
+                .store(std::ptr::null_mut(), std::sync::atomic::Ordering::Relaxed);
+        }
+    }
+
     /// Compute a fresh locals mapping from the live frame state —
     /// the class/module namespace itself for those scopes, or a
     /// fast-locals snapshot dict for function scopes. This replaces
@@ -1163,6 +1215,16 @@ impl<T> std::ops::Deref for FrameSlot<T> {
     }
 }
 
+impl Drop for FrameShell {
+    fn drop(&mut self) {
+        // A frame object outliving its shell (a spine dropped whole) must
+        // not read through a dead link.
+        if let Some(py) = self.materialized.get_mut().take() {
+            py.unlink_live(self);
+        }
+    }
+}
+
 impl FrameShell {
     /// Make every borrowed slot own its value (see [`FrameSlot`]).
     pub(crate) fn upgrade_slots(&self) {
@@ -1199,14 +1261,8 @@ impl FrameShell {
         }
     }
 
-    /// Build the real [`PyFrame`] for this shell with the given
-    /// `back` pointer, caching it. Bumps `on_stack` exactly once per
-    /// materialisation — the pop path decrements it for shells whose
-    /// `materialized` is set.
-    /// Bring an existing frame object's `lasti` up to this live shell's.
-    /// The executing frame's quiet loops sync only the shell while nothing
-    /// else holds the object (see `Interpreter::frame_object_observed`);
-    /// every path that hands the object out calls this first. A trace
+    /// Bring an existing frame object's `lasti` up to this shell's: the
+    /// dispatch loops sync only the shell (see [`PyFrame::live`]). A trace
     /// function's pending line jump owns the value and is left alone.
     pub fn refresh_materialized(&self, py: &PyFrame) {
         if py.pending_jump.borrow().is_none() && py.override_lineno.get().is_none() {
@@ -1215,23 +1271,26 @@ impl FrameShell {
         }
     }
 
+    /// Make `py` read its `lasti` from this shell, which is on its spine
+    /// and keeps `py` as its frame object (see [`PyFrame::live`]).
+    pub(crate) fn link(&self, py: &PyFrame) {
+        py.live.store(
+            std::ptr::from_ref(self).cast_mut(),
+            std::sync::atomic::Ordering::Relaxed,
+        );
+    }
+
+    /// Build the real [`PyFrame`] for this shell, which is on its spine,
+    /// with the given `back` pointer, caching it. Bumps `on_stack` exactly
+    /// once per materialisation — the pop path decrements it for shells
+    /// whose `materialized` is set.
     pub fn materialize(&self, back: Option<Rc<PyFrame>>) -> Rc<PyFrame> {
-        let py = self.materialize_departing(back);
-        // RFC 0065 (WS1): a freshly materialized frame must kick its
-        // dispatch loop off the quiet path so the per-instruction
-        // `lasti`-cell sync resumes.
-        crate::hot_gates::bump_loop_gen();
-        py
+        self.materialize_departing(back)
     }
 
-    /// [`Self::materialize`] for an activation that is leaving the spine:
-    /// no dispatch loop will run it again, so none needs kicking off its
-    /// quiet path.
+    /// [`Self::materialize`] for an activation that is leaving the spine.
     pub fn materialize_departing(&self, back: Option<Rc<PyFrame>>) -> Rc<PyFrame> {
         if let Some(existing) = self.materialized.borrow().as_ref() {
-            // (`materialize` still bumps: the new holder may keep it, and
-            // a quiet loop running this activation re-derives, finding it
-            // observed if so.)
             self.refresh_materialized(existing);
             return existing.clone();
         }
@@ -1259,7 +1318,9 @@ impl FrameShell {
             extra_locals: RefCell::new(None),
             cleared: Cell::new(false),
             lazy_back: RefCell::new(None),
+            live: std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()),
         });
+        self.link(&py);
         *self.materialized.borrow_mut() = Some(py.clone());
         self.has_materialized
             .store(true, std::sync::atomic::Ordering::Release);
@@ -1326,13 +1387,12 @@ pub fn materialize_stack_at(stack: &FrameStack, idx: usize) -> Option<Rc<PyFrame
         return None;
     }
     let mut back: Option<Rc<PyFrame>> = None;
-    let mut refreshed = false;
     for shell in &s[..=idx] {
         let existing = shell.materialized.borrow().clone();
         let py = match existing {
             Some(py) => {
                 shell.refresh_materialized(&py);
-                refreshed = true;
+                shell.link(&py);
                 py.set_back(back);
                 py
             }
@@ -1344,11 +1404,6 @@ pub fn materialize_stack_at(stack: &FrameStack, idx: usize) -> Option<Rc<PyFrame
         };
         back = Some(py);
     }
-    if refreshed {
-        // See `FrameShell::materialize`: the objects handed out (the
-        // frame and its `f_back` chain) may be kept.
-        crate::hot_gates::bump_loop_gen();
-    }
     back
 }
 
@@ -1368,10 +1423,10 @@ pub fn materialize_stack_at_lazy(stack: &FrameStack, idx: usize) -> Option<Rc<Py
         return materialize_stack_at(stack, idx);
     }
     let existing = shell.materialized.borrow().clone();
-    let mut refreshed = existing.is_some();
     let py = match existing {
         Some(py) => {
             shell.refresh_materialized(&py);
+            shell.link(&py);
             py
         }
         // Counted as live on the stack, as the eager walk counts it.
@@ -1384,7 +1439,7 @@ pub fn materialize_stack_at_lazy(stack: &FrameStack, idx: usize) -> Option<Rc<Py
             match caller_py {
                 Some(back) => {
                     c.refresh_materialized(&back);
-                    refreshed = true;
+                    c.link(&back);
                     let linked = py.lazy_back.borrow().is_none()
                         && py
                             .back
@@ -1410,10 +1465,6 @@ pub fn materialize_stack_at_lazy(stack: &FrameStack, idx: usize) -> Option<Rc<Py
             }
         }
     }
-    if refreshed {
-        // As in `materialize_stack_at`: a new holder of existing objects.
-        crate::hot_gates::bump_loop_gen();
-    }
     Some(py)
 }
 
diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs
index 2262e0df..f10edb38 100644
--- a/crates/weavepy-vm/src/stdlib/sys.rs
+++ b/crates/weavepy-vm/src/stdlib/sys.rs
@@ -54,8 +54,8 @@ pub fn build_with_state(
                 name: "_getframe",
                 binds_instance: false,
                 call: Box::new(move |args| {
-                    if let Some(h) = crate::vm_singletons::current_thread_handles() {
-                        sys_getframe(args, &h.frame_stack)
+                    if let Some(fs) = crate::vm_singletons::current_frame_stack() {
+                        sys_getframe(args, &fs)
                     } else {
                         sys_getframe(args, &fs_fallback)
                     }
@@ -70,8 +70,8 @@ pub fn build_with_state(
                 name: "_getframemodulename",
                 binds_instance: false,
                 call: Box::new(move |args| {
-                    if let Some(h) = crate::vm_singletons::current_thread_handles() {
-                        sys_getframemodulename(args, &h.frame_stack)
+                    if let Some(fs) = crate::vm_singletons::current_frame_stack() {
+                        sys_getframemodulename(args, &fs)
                     } else {
                         sys_getframemodulename(args, &fs_fallback_modname)
                     }
@@ -1589,7 +1589,9 @@ fn sys_getframe(
     match crate::object::materialize_stack_at_lazy(frame_stack, idx) {
         Some(py) => {
             // PEP 578: `sys._getframe` audits with the frame object.
-            audit_event("sys._getframe", &[Object::Frame(py.clone())])?;
+            if crate::trace::any_audit_active() {
+                audit_event("sys._getframe", &[Object::Frame(py.clone())])?;
+            }
             Ok(Object::Frame(py))
         }
         None => Err(value_error("call stack is not deep enough")),
diff --git a/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs b/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs
index f3129168..65eb4638 100644
--- a/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs
@@ -209,6 +209,7 @@ fn frame_new(args: &[Object]) -> Result<Object, RuntimeError> {
         extra_locals: RefCell::new(None),
         cleared: Cell::new(false),
         lazy_back: RefCell::new(None),
+        live: std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()),
     });
     Ok(Object::Frame(frame))
 }
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index 33359819..23df33d4 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -4667,6 +4667,10 @@ struct CallCtx {
     /// activation shell the inspection found (a framed activation's are
     /// its frame's own).
     inspected_locals: std::cell::RefCell<Option<Rc<GilRefCell<Vec<Object>>>>>,
+    /// A frameless activation's frame object, made for its activation
+    /// shell while a call ran (`sys._getframe()`): the interpreter
+    /// continuation adopts it (see [`call_with_activation_shell`]).
+    inspected_frame: std::cell::RefCell<Option<Rc<crate::object::PyFrame>>>,
 }
 
 /// A framed native activation running on this thread, registered so
@@ -6463,6 +6467,7 @@ fn fresh_child(ctx: &CallCtx, nc: &NativeCallee, callee_key: *const CodeObject)
         pin_memo: [u32::MAX; PIN_MEMO],
         introspected: Cell::new(false),
         inspected_locals: std::cell::RefCell::new(None),
+        inspected_frame: std::cell::RefCell::new(None),
         cell_list_pins: Vec::new(),
     })
 }
@@ -6840,9 +6845,23 @@ fn finish_deopted(
     // An inspected activation's shell holds its locals, plus whatever the
     // inspection wrote (see `sync_native_locals`).
     let inspected = nctx.inspected_locals.borrow_mut().take();
+    // A frame object made for the activation's shell stands for this
+    // frame: the frame runs on in it, on the storage its `f_locals` reads.
+    let adopted = nctx.inspected_frame.borrow_mut().take();
+    let mut shared_locals = None;
     if let Some(shell) = inspected.filter(|_| nctx.introspected.get()) {
-        locals_v.extend(shell.borrow().iter().take(n_real).cloned());
-        locals_v.resize(n_real, Object::Unbound);
+        if adopted.as_ref().is_some_and(|py| {
+            py.locals_mirror
+                .borrow()
+                .as_ref()
+                .is_some_and(|m| Rc::ptr_eq(m, &shell))
+        }) {
+            shell.borrow_mut().resize(n_real, Object::Unbound);
+            shared_locals = Some(shell);
+        } else {
+            locals_v.extend(shell.borrow().iter().take(n_real).cloned());
+            locals_v.resize(n_real, Object::Unbound);
+        }
     } else {
         for slot in 0..n_real {
             match entry.cf.local_types.get(slot).copied().flatten() {
@@ -6855,9 +6874,10 @@ fn finish_deopted(
             }
         }
     }
+    let py_frame = shared_locals.as_ref().and(adopted);
     let mut frame = super::Frame {
         code: code.clone(),
-        locals: Rc::new(GilRefCell::new(locals_v)),
+        locals: shared_locals.unwrap_or_else(|| Rc::new(GilRefCell::new(locals_v))),
         cells: crate::object::empty_cells(),
         stack: Vec::new(),
         globals: func.globals.clone(),
@@ -6866,7 +6886,7 @@ fn finish_deopted(
         exc: None,
         agen_yielded_value: true,
         pc: 0,
-        py_frame: None,
+        py_frame,
         gen_owner: None,
         suppress_call_event: true,
         gen_first_resume: false,
@@ -6960,9 +6980,26 @@ fn call_with_activation_shell<T>(
         materialized: GilRefCell::new(None),
         tb_refs: std::sync::atomic::AtomicU32::new(0),
     });
-    interp.frame_stack.borrow_mut().push(shell);
+    interp.frame_stack.borrow_mut().push(shell.clone());
     // Inspection of the shell's frame finds this activation's locals.
-    let out = with_native_frame(&shell_locals, jf, || f(interp));
+    let out = with_native_frame(&shell_locals, jf, || {
+        let out = f(interp);
+        // The call made the activation's frame object (`sys._getframe()`
+        // in a callee, or the native code's own call of it): one object
+        // stands for the activation from now on, so the rest of it runs
+        // in the interpreter (the inspection mark ends the native run
+        // after this call), whose frame adopts the object and these
+        // locals (see `finish_deopted`).
+        // (An inspection of its `f_locals` synced them already, and may
+        // have written them since.)
+        if let Some(py) = shell.materialized.borrow().as_ref() {
+            if !ctx.introspected.get() {
+                sync_native_locals(&shell_locals);
+            }
+            *ctx.inspected_frame.borrow_mut() = Some(py.clone());
+        }
+        out
+    });
     // The general pop: a traceback entry or a callee frame's `f_back` may
     // hold the shell lazily, and gets its frame object on the way out.
     interp.pop_frame_shell();
@@ -13772,6 +13809,7 @@ pub(crate) fn try_call_native_direct(
         pin_memo: [u32::MAX; PIN_MEMO],
         introspected: Cell::new(false),
         inspected_locals: std::cell::RefCell::new(None),
+        inspected_frame: std::cell::RefCell::new(None),
         cell_list_pins: Vec::new(),
     };
     let mut jf = JitFrame {
@@ -14729,6 +14767,7 @@ fn enter_compiled(
         pin_memo: [u32::MAX; PIN_MEMO],
         introspected: Cell::new(false),
         inspected_locals: std::cell::RefCell::new(None),
+        inspected_frame: std::cell::RefCell::new(None),
         cell_list_pins: Vec::new(),
     };
     let mut jf = JitFrame {
@@ -15650,6 +15689,7 @@ fn resume_parked(interp: &mut super::Interpreter, frame: &mut super::Frame) -> J
         pin_memo: [u32::MAX; PIN_MEMO],
         introspected: Cell::new(false),
         inspected_locals: std::cell::RefCell::new(None),
+        inspected_frame: std::cell::RefCell::new(None),
         cell_list_pins: Vec::new(),
     };
     let mut jf = JitFrame {
diff --git a/crates/weavepy-vm/src/vm_singletons.rs b/crates/weavepy-vm/src/vm_singletons.rs
index 47e66099..53f6f668 100644
--- a/crates/weavepy-vm/src/vm_singletons.rs
+++ b/crates/weavepy-vm/src/vm_singletons.rs
@@ -668,6 +668,12 @@ pub fn current_thread_handles() -> Option<ThreadHandles> {
     CURRENT_THREAD_HANDLES.with(|cell| cell.borrow().last().cloned())
 }
 
+/// The current thread's frame stack alone (see [`current_thread_handles`],
+/// whose other handles `sys._getframe` and its kin never read).
+pub fn current_frame_stack() -> Option<crate::object::FrameStack> {
+    CURRENT_THREAD_HANDLES.with(|cell| cell.borrow().last().map(|h| h.frame_stack.clone()))
+}
+
 /// RFC 0066 WS4 — swap the *entire* handles stack, returning the old
 /// one. Greenlet switches use this: handle guards live on whichever
 /// native stack pushed them, so once greenlets multiplex several
diff --git a/tests/regrtest/test_frame_live_lineno.py b/tests/regrtest/test_frame_live_lineno.py
new file mode 100644
index 00000000..97ad1d6f
--- /dev/null
+++ b/tests/regrtest/test_frame_live_lineno.py
@@ -0,0 +1,175 @@
+"""A frame object of a live activation reports the line its activation
+is at, however the activation runs (interpreted, inline, or compiled once
+hot), and a frame object outliving its activation keeps the line it left
+at. Each check runs cold and again once its loops and callees are hot."""
+
+import sys
+
+N = 30000
+
+with open(__file__) as _f:
+    _SRC = _f.read().splitlines()
+
+
+def L(tag):
+    """The line carrying the marker comment `tag`."""
+    marker = "# " + tag
+    (line,) = [i + 1 for i, s in enumerate(_SRC) if s.endswith(marker)]
+    return line
+
+
+def self_reads(n):
+    fr = sys._getframe()
+    bad = 0
+    for i in range(n):
+        a = fr.f_lineno  # sr-a
+        x = i + 1
+        b = fr.f_lineno  # sr-b
+        if (a, b) != (L("sr-a"), L("sr-b")):
+            bad += 1
+    return bad
+
+
+def self_getattr(n):
+    fr = sys._getframe()
+    seen = set()
+    for i in range(n):
+        seen.add(getattr(fr, "f_lineno"))  # sg-a
+        seen.add(getattr(fr, "f_lineno"))  # sg-b
+    return sorted(seen)
+
+
+def keep():
+    return sys._getframe(1)
+
+
+def held_by_callee_then_read(a):
+    fr = keep()
+    x = a + 1
+    first = fr.f_lineno  # hc-a
+    y = x * 2
+    return fr, first, fr.f_lineno  # hc-b
+
+
+F = None
+
+
+def stores_own_frame(a):
+    global F
+    F = sys._getframe()
+    x = a + 1
+    return x  # so-ret
+
+
+def returns_own_frame(a):
+    fr = sys._getframe()
+    x = a + 1
+    y = x * 2
+    return fr  # ro-ret
+
+
+def walk():
+    f = sys._getframe(1)
+    out = []
+    while f is not None and f.f_code.co_name != "<module>":
+        out.append((f.f_code.co_name, f.f_lineno))
+        f = f.f_back
+    return out
+
+
+def d1():
+    return walk()  # d1-call
+
+
+def d2():
+    r = d1()  # d2-call
+    return r
+
+
+class Getattr:
+    def __getattr__(self, name):
+        return sys._getframe(1).f_lineno
+
+
+class Prop:
+    @property
+    def line(self):
+        return sys._getframe(1).f_lineno
+
+
+def escapes(n):
+    g, p = Getattr(), Prop()
+    seen = set()
+    for i in range(n):
+        seen.add(g.anything)  # es-a
+        z = 0
+        seen.add(p.line)  # es-b
+        sorted([2, 1], key=lambda v: seen.add(sys._getframe(1).f_lineno) or v)  # es-c
+    return sorted(seen)
+
+
+def raiser(a):
+    if a >= 0:
+        raise ValueError(a)  # ra-raise
+
+
+def catches(n):
+    lines = set()
+    for i in range(n):
+        try:
+            raiser(i)  # ca-call
+        except ValueError as e:
+            tb = e.__traceback__
+            lines.add((tb.tb_lineno, tb.tb_frame.f_lineno))  # ca-add
+            lines.add((tb.tb_next.tb_lineno, tb.tb_next.tb_frame.f_lineno))
+    return sorted(lines)
+
+
+def gen():
+    yield 1  # ge-a
+    x = 2
+    yield x  # ge-b
+
+
+def gen_lines(n):
+    seen = set()
+    for i in range(n):
+        g = gen()
+        next(g)
+        seen.add(g.gi_frame.f_lineno)
+        next(g)
+        seen.add(g.gi_frame.f_lineno)
+    return sorted(seen)
+
+
+def run(n):
+    assert self_reads(n) == 0
+    got = self_getattr(n)
+    assert got == [L("sg-a"), L("sg-b")], got
+    seen = set()
+    for i in range(n):
+        fr, first, second = held_by_callee_then_read(i)
+        seen.add((first, second, fr.f_lineno, fr.f_back.f_code.co_name))
+        stores_own_frame(i)
+        seen.add(("F", F.f_lineno))
+        seen.add(("R", returns_own_frame(i).f_lineno))
+    want = {
+        (L("hc-a"), L("hc-b"), L("hc-b"), "run"),
+        ("F", L("so-ret")),
+        ("R", L("ro-ret")),
+    }
+    assert seen == want, seen
+    for i in range(n // 10):
+        w = d2()  # run-d2
+    assert w == [("d1", L("d1-call")), ("d2", L("d2-call")), ("run", L("run-d2"))], w
+    got = escapes(n)
+    assert got == [L("es-a"), L("es-b"), L("es-c")], got
+    got = catches(n // 10)
+    assert got == [(L("ra-raise"), L("ra-raise")), (L("ca-call"), L("ca-add"))], got
+    got = gen_lines(n // 10)
+    assert got == [L("ge-a"), L("ge-b")], got
+
+
+run(10)
+run(N)
+run(N)

From b724fba6ca80ccd1168ad584b6c7b98fe8859946 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 04:27:44 -0700
Subject: [PATCH 041/173] perf: read frame fields in line and flush only the
 frames asked for

- Frame and code object fields (f_code, f_back, f_globals, co_name, ...)
  load in the frame JIT's in-line attribute helpers and ahead of the
  property and descriptor probes in the core loop's attribute lane. A
  live frame's f_lineno is its shell's lasti, which is current unless
  the frame is the running activation's own (its frame object's locals
  mirror is the running frame's storage): then the line is that of the
  instruction running, with no trip through the pending list.
- sys._getframe(n) pushes shells only for the lean activations up to
  the one it returns; the newer ones stay lean.
- sys._getframemodulename(n) reads the globals of the activation n
  levels up wherever it is, lean or on the spine, and pushes no shells.

Measured (instructions per call, pinned seed): logging findCaller
16.7k -> 13.1k (CPython 5.9k); frame.f_lineno of a caller's frame from
a compiled loop 2.4k -> 0.85k (CPython 1.0k); a frame reading its own
f_lineno 2.5k -> 0.9k; frame.f_code.co_argcount 0.8k -> 0.4-0.6k;
sys._getframemodulename(1) from a call chain 5.3k -> 3.7k; one log.info
151k -> 146k. Suite: logging_mix -2.1%, geomean unchanged within
layout noise (1.0011x vs the previous commit, unrelated workloads
+/-0.9%).
---
 crates/weavepy-vm/src/frame_jit.rs       |  29 ++++++
 crates/weavepy-vm/src/lib.rs             | 110 ++++++++++++++++++-----
 crates/weavepy-vm/src/stdlib/sys.rs      |  26 +++++-
 tests/regrtest/test_frame_live_lineno.py |  38 ++++++++
 4 files changed, 178 insertions(+), 25 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 40826757..f6585ef5 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3555,6 +3555,10 @@ unsafe extern "C" fn h_stack_attr(
             Object::Type(cls) if Rc::strong_count(cls) > 1 => type_attr(code, ext, cls, pc, arg),
             Object::Module(m) => Interpreter::core_module_attr(code, m, pc, arg)
                 .or_else(|| Interpreter::leaf_fused_local_attr(code, recv, pc, arg)),
+            Object::Code(_) | Object::Frame(_) => code
+                .names
+                .get(arg as usize)
+                .and_then(|name| Interpreter::frame_plain_attr(recv, name, None)),
             _ => None,
         };
         let Some(v) = v else {
@@ -3625,6 +3629,12 @@ unsafe extern "C" fn h_local_attr(
                 Interpreter::core_local_attr(Some(ext), code, recv, pc, arg)
             }
             Object::Type(cls) => type_attr(code, ext, cls, pc, arg),
+            // A frame's or code object's plain field (a live frame's line
+            // needs the full lane, which makes its `lasti` current).
+            Object::Code(_) | Object::Frame(_) => code
+                .names
+                .get(arg as usize)
+                .and_then(|name| Interpreter::frame_plain_attr(recv, name, None)),
             _ => None,
         };
         match v {
@@ -4068,6 +4078,25 @@ unsafe extern "C" fn h_property(
             return CALL_DECLINED;
         }
         let (pc, slot) = (pc as usize, slot as usize);
+        // A frame's or code object's plain field, a live frame's line with
+        // the running activation's own instruction.
+        if matches!(&*recv, Object::Frame(_) | Object::Code(_)) && (local != 0 || droppable(&*recv))
+        {
+            let code = &*code;
+            let running = (Rc::as_ptr(&(*st.frame).locals).cast::<()>(), pc as u32);
+            let v = code
+                .names
+                .get(code.instructions[pc].arg as usize)
+                .and_then(|name| Interpreter::frame_plain_attr(&*recv, name, Some(running)));
+            if let Some(v) = v {
+                let at = st.stack.add(slot);
+                if local == 0 {
+                    crate::drop_hot(at.read());
+                }
+                at.write(v);
+                return 0;
+            }
+        }
         let Object::Instance(inst) = &*recv else {
             return attr_lane(st, pc, slot, recv, local);
         };
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 0b2064b8..162cc320 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -10547,6 +10547,46 @@ impl Interpreter {
         self.run_frame(frame)
     }
 
+    /// Make the activation `depth` levels up the whole stack (the spine,
+    /// then the pending lean activations, which are always newer than
+    /// every shell on it) the spine's, pushing the shells of the pending
+    /// ones up to it and leaving the newer ones pending. Returns its depth
+    /// from the spine's top.
+    pub(crate) fn flush_pending_through(&mut self, depth: usize) -> usize {
+        let n = self.lean_pending.len();
+        if depth >= n {
+            return depth - n;
+        }
+        let mut pending = std::mem::take(&mut self.lean_pending);
+        for addr in &pending[..n - depth] {
+            // SAFETY: see `flush_lean_cold`.
+            let caller = unsafe { &mut *(*addr as *mut LeanAct) };
+            if caller.shell.is_none() {
+                caller.shell = Some(self.push_pending_caller_shell(caller));
+            }
+        }
+        pending.drain(..n - depth);
+        self.lean_pending = pending;
+        0
+    }
+
+    /// The globals of the activation `depth` levels up the whole stack
+    /// (see [`Self::flush_pending_through`]), without pushing any shell.
+    pub(crate) fn frame_globals_at(&self, depth: usize) -> Option<Rc<RefCell<DictData>>> {
+        let n = self.lean_pending.len();
+        if depth < n {
+            // SAFETY: a pending address names a live `LeanAct` whose
+            // frame pointer its `lean_pending_enter` re-derived (see
+            // `flush_lean_cold`).
+            let act = unsafe { &*(self.lean_pending[n - 1 - depth] as *const LeanAct) };
+            // SAFETY: as above.
+            return Some(unsafe { (*act.frame).globals.clone() });
+        }
+        let stack = self.frame_stack.borrow();
+        let idx = stack.len().checked_sub(1 + depth - n)?;
+        Some((*stack[idx].globals).clone())
+    }
+
     /// Push the shells of every pending lean *caller* (see
     /// [`Self::flush_lean_cold`], which also pushes the running one's).
     pub(crate) fn flush_pending_callers(&mut self) {
@@ -14330,7 +14370,8 @@ impl Interpreter {
                             // object's field (the helper's plain values).
                             _ => {
                                 // SAFETY: `top` is the receiver's slot.
-                                if unsafe { self.core_plain_attr(code, ins.arg, top) } {
+                                let running = (Rc::as_ptr(&frame.locals).cast::<()>(), pc as u32);
+                                if unsafe { self.core_plain_attr(code, ins.arg, top, running) } {
                                     last = pc;
                                     pc += 1;
                                     after_release!();
@@ -18284,13 +18325,19 @@ impl Interpreter {
     ///
     /// `top` is the core loop's top stack slot.
     #[inline(never)]
-    unsafe fn core_plain_attr(&self, code: &CodeObject, name_idx: u32, top: *mut Object) -> bool {
+    unsafe fn core_plain_attr(
+        &self,
+        code: &CodeObject,
+        name_idx: u32,
+        top: *mut Object,
+        running: (*const (), u32),
+    ) -> bool {
         // SAFETY: the caller's contract.
         let recv = unsafe { &*top };
         if !Self::core_droppable(recv) {
             return false;
         }
-        let Some(v) = self.leaf_plain_attr_value(code, Some(recv), name_idx) else {
+        let Some(v) = self.leaf_plain_attr_value(code, Some(recv), name_idx, Some(running)) else {
             return false;
         };
         // SAFETY: the receiver (droppable) is replaced in place.
@@ -21009,7 +21056,9 @@ impl Interpreter {
         // `y.append` (a builtin container's method read as a value) and
         // `obj.__dict__`: sites the full handler never specializes.
         if ins.op == OpCode::LoadAttr {
-            if let Some(v) = self.leaf_plain_attr_value(code, stack.last(), ins.arg) {
+            let running = (Rc::as_ptr(&frame.locals).cast::<()>(), pc as u32);
+            if let Some(v) = self.leaf_plain_attr_value(code, stack.last(), ins.arg, Some(running))
+            {
                 if let Some(top) = stack.last_mut() {
                     drop(std::mem::replace(top, v));
                 }
@@ -21243,11 +21292,12 @@ impl Interpreter {
         code: &CodeObject,
         recv: Option<&Object>,
         name_idx: u32,
+        running: Option<(*const (), u32)>,
     ) -> Option<Object> {
         let recv = recv?;
         let name = code.names.get(name_idx as usize)?.as_str();
         match recv {
-            Object::Code(_) | Object::Frame(_) => Self::frame_plain_attr(recv, name, false),
+            Object::Code(_) | Object::Frame(_) => Self::frame_plain_attr(recv, name, running),
             // `x.__class__` of a builtin value: its exact type (no instance
             // or class can override it there).
             Object::Generator(_)
@@ -21275,21 +21325,36 @@ impl Interpreter {
     /// A code object's fields and a frame's plain ones (neither type can
     /// be subclassed, and no getter here runs Python); the full path keeps
     /// the deprecated `co_lnotab`, a generator frame's re-derived
-    /// `f_back`, and a line-0 `f_lineno`. A live frame's line needs its
-    /// activation's `lasti` current (see `PyFrame::live`), which the
-    /// running one's is only once it waits on the pending list: `synced`
-    /// says it does.
+    /// `f_back`, and a line-0 `f_lineno`. A live frame's line is its
+    /// shell's `lasti` (see `PyFrame::live`), which is current unless its
+    /// activation is the one running: `running` names the running
+    /// activation (its locals' storage, which its frame object's mirror
+    /// shares) and the instruction it is at, if the caller knows them.
     #[inline]
-    fn frame_plain_attr(recv: &Object, name: &str, synced: bool) -> Option<Object> {
+    fn frame_plain_attr(
+        recv: &Object,
+        name: &str,
+        running: Option<(*const (), u32)>,
+    ) -> Option<Object> {
         match recv {
             Object::Code(c) if name != "co_lnotab" => crate::builtins::code_synthetic_attr(c, name),
             Object::Frame(fr) => match name {
                 "f_code" => Some(Object::Code(fr.code.clone())),
                 "f_globals" => Some(Object::Dict(fr.globals.clone())),
                 "f_builtins" => Some(Object::Dict(fr.builtins.clone())),
-                "f_lineno"
-                    if synced || fr.live.load(std::sync::atomic::Ordering::Relaxed).is_null() =>
-                {
+                "f_lineno" => {
+                    if !fr.live.load(std::sync::atomic::Ordering::Relaxed).is_null() {
+                        let (locals, pc) = running?;
+                        let own = fr
+                            .locals_mirror
+                            .try_borrow()
+                            .ok()?
+                            .as_ref()
+                            .is_some_and(|m| std::ptr::eq(Rc::as_ptr(m).cast::<()>(), locals));
+                        if own {
+                            fr.set_lasti(pc);
+                        }
+                    }
                     match fr.current_lineno() {
                         0 => None,
                         line => Some(Object::Int(i64::from(line))),
@@ -46096,9 +46161,12 @@ impl Interpreter {
         // SAFETY: see `CoreSwitch`: the running activation is synced and
         // unborrowed here.
         let frame = unsafe { &mut *sw.cur };
+        let frame_field = frame.code.instructions[pc].op == OpCode::LoadAttr
+            && matches!(frame.stack.last(), Some(Object::Frame(_) | Object::Code(_)));
         // A property's or a Python data descriptor's getter runs as an
         // inline activation, as the core loop's own arm runs it.
         if frame.code.instructions[pc].op == OpCode::LoadAttr
+            && !frame_field
             && (self.core_property(sw) || self.core_getitem(sw, true))
         {
             return Ok(());
@@ -46106,26 +46174,24 @@ impl Interpreter {
         // SAFETY: as above (both declined untouched).
         let frame = unsafe { &mut *sw.cur };
         let ins = frame.code.instructions[pc];
-        frame.pc = pc as u32 + 1;
-        let pending = self.core_pending_enter(sw, frame, pc);
-        // A frame's or code object's plain field, now that this
-        // activation's `lasti` is current for its own frame's line.
-        if ins.op == OpCode::LoadAttr
-            && matches!(frame.stack.last(), Some(Object::Frame(_) | Object::Code(_)))
-        {
+        // A frame's or code object's plain field.
+        if frame_field {
+            let running = (Rc::as_ptr(&frame.locals).cast::<()>(), pc as u32);
             let v = match (frame.stack.last(), frame.code.names.get(ins.arg as usize)) {
-                (Some(recv), Some(name)) => Self::frame_plain_attr(recv, name, true),
+                (Some(recv), Some(name)) => Self::frame_plain_attr(recv, name, Some(running)),
                 _ => None,
             };
             if let Some(v) = v {
+                frame.pc = pc as u32 + 1;
                 if let Some(top) = frame.stack.last_mut() {
                     let recv = std::mem::replace(top, v);
                     self.release(recv);
                 }
-                self.core_pending_exit(sw, pending);
                 return Ok(());
             }
         }
+        frame.pc = pc as u32 + 1;
+        let pending = self.core_pending_enter(sw, frame, pc);
         let r = match ins.op {
             OpCode::LoadAttr => match self.load_attr_certain_miss(frame, ins.arg) {
                 Some(err) => Err(err),
diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs
index f10edb38..2e913f34 100644
--- a/crates/weavepy-vm/src/stdlib/sys.rs
+++ b/crates/weavepy-vm/src/stdlib/sys.rs
@@ -1564,18 +1564,25 @@ fn sys_getframe(
     args: &[Object],
     frame_stack: &crate::object::FrameStack,
 ) -> Result<Object, RuntimeError> {
-    crate::builtins::sync_frame_spine();
     if args.len() > 1 {
         return Err(type_error(format!(
             "_getframe expected at most 1 argument, got {}",
             args.len()
         )));
     }
-    let depth = match args.first() {
+    let mut depth = match args.first() {
         Some(Object::Int(d)) => *d as usize,
         None => 0,
         _ => return Err(type_error("depth must be an int")),
     };
+    // Only the activations up to the one asked for need their shells: the
+    // newer ones stay lean.
+    match crate::builtins::reentrant_interp() {
+        Ok(interp) if Rc::ptr_eq(&interp.frame_stack, frame_stack) => {
+            depth = interp.flush_pending_through(depth);
+        }
+        _ => crate::builtins::sync_frame_spine(),
+    }
     // The topmost frame is the currently-executing one, which is
     // the *callee* of `sys._getframe`. CPython considers the
     // calling frame as depth 0; we mirror by indexing from the back.
@@ -1606,7 +1613,6 @@ fn sys_getframemodulename(
     args: &[Object],
     frame_stack: &crate::object::FrameStack,
 ) -> Result<Object, RuntimeError> {
-    crate::builtins::sync_frame_spine();
     let depth = match args.first() {
         Some(o) => match o {
             Object::Int(d) => *d,
@@ -1618,6 +1624,20 @@ fn sys_getframemodulename(
     if depth < 0 {
         return Ok(Object::None);
     }
+    // The globals are read where the activation is, lean or not.
+    if let Ok(interp) = crate::builtins::reentrant_interp() {
+        if Rc::ptr_eq(&interp.frame_stack, frame_stack) {
+            let Some(globals) = interp.frame_globals_at(depth as usize) else {
+                return Ok(Object::None);
+            };
+            let name = globals
+                .borrow()
+                .get(&crate::object::StrKey("__name__"))
+                .cloned();
+            return Ok(name.unwrap_or(Object::None));
+        }
+    }
+    crate::builtins::sync_frame_spine();
     let len = frame_stack.borrow().len();
     let depth = depth as usize;
     if depth >= len {
diff --git a/tests/regrtest/test_frame_live_lineno.py b/tests/regrtest/test_frame_live_lineno.py
index 97ad1d6f..0bad8cd1 100644
--- a/tests/regrtest/test_frame_live_lineno.py
+++ b/tests/regrtest/test_frame_live_lineno.py
@@ -142,7 +142,45 @@ def gen_lines(n):
     return sorted(seen)
 
 
+import types
+
+M = types.ModuleType("modx")
+exec(
+    "import sys\n"
+    "def who(d):\n"
+    "    return sys._getframemodulename(d)\n"
+    "def via(d):\n"
+    "    return who(d)\n"
+    "def frame_at(d):\n"
+    "    return sys._getframe(d)\n"
+    "def chain(d):\n"
+    "    return frame_at(d)\n",
+    M.__dict__,
+)
+HERE = __name__
+
+
+def stack_names(n):
+    seen = set()
+    for i in range(n):
+        seen.add((M.who(0), M.who(1), M.via(1), M.via(2), M.via(10**6)))
+        f = M.chain(1)
+        names = []
+        while f is not None and f.f_code.co_name != "<module>":
+            names.append(f.f_code.co_name)
+            f = f.f_back
+        seen.add(tuple(names))
+        try:
+            M.chain(10**6)
+        except ValueError:
+            seen.add("deep")
+    return seen
+
+
 def run(n):
+    want = {("modx", HERE, "modx", HERE, None), ("chain", "stack_names", "run"), "deep"}
+    got = stack_names(n)
+    assert got == want, got
     assert self_reads(n) == 0
     got = self_getattr(n)
     assert got == [L("sg-a"), L("sg-b")], got

From 77818a4ead15bfe74ae98f96f808c7cb8eb9ee76 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 05:18:50 -0700
Subject: [PATCH 042/173] perf: recycle frame shells whose frame object died,
 keep code names

- A returning activation's shell whose frame object nothing else holds
  (a sys._getframe handle since dropped, as in logging's findCaller)
  drops the object and goes back to the shell pool, instead of being
  freed with it.
- co_name, co_qualname and co_filename return one str object per code
  object (kept in its VM extension) instead of a new string per read.
- The frame JIT's in-line frame and code field reads skip the instance
  slot cache bookkeeping, which only applies to instances.
- StringIO.write appending at the end with nothing to translate or
  encode appends the text directly, without the codec round trip and
  UTF-8 revalidation (StreamHandler.emit's shape).

Measured (instructions per call, pinned seed): sys._getframe(1) from a
call chain 7.9k -> 6.3k; logging findCaller 13.1k -> 11.5k; one log.info
144k -> 141k. Suite geomean 0.9993x vs the previous commit
(logging_mix -2.2%; closures_decorators +2.6%, which a build with only
the StringIO change shows too: code layout, not these paths).
---
 crates/weavepy-vm/src/builtins.rs  | 32 +++++++++++++++++-------
 crates/weavepy-vm/src/frame_jit.rs | 32 ++++++++++++++++++------
 crates/weavepy-vm/src/lib.rs       | 33 +++++++++++++++++++-----
 crates/weavepy-vm/src/object.rs    | 40 ++++++++++++++++++++++++++++++
 4 files changed, 114 insertions(+), 23 deletions(-)

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index e1cf5d0b..0dea36d5 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -4306,14 +4306,26 @@ pub(crate) fn code_synthetic_attr(
     c: &Rc<weavepy_compiler::CodeObject>,
     name: &str,
 ) -> Option<Object> {
+    let ident = |k: usize| {
+        crate::code_vm_ext(c).map(|ext| {
+            ext.ident_strs.get_or_init(|| {
+                [
+                    Object::from_str(&c.name),
+                    Object::from_str(if c.qualname.is_empty() {
+                        &c.name
+                    } else {
+                        &c.qualname
+                    }),
+                    Object::from_str(&*c.filename),
+                ]
+            })[k]
+                .clone()
+        })
+    };
     match name {
-        "co_name" | "__name__" => Some(Object::from_str(&c.name)),
-        "co_qualname" | "__qualname__" => Some(Object::from_str(if c.qualname.is_empty() {
-            &c.name
-        } else {
-            &c.qualname
-        })),
-        "co_filename" => Some(Object::from_str(&*c.filename)),
+        "co_name" | "__name__" => ident(0),
+        "co_qualname" | "__qualname__" => ident(1),
+        "co_filename" => ident(2),
         "co_argcount" => Some(Object::Int(i64::from(c.arg_count))),
         "co_posonlyargcount" => Some(Object::Int(i64::from(c.posonly_count))),
         "co_kwonlyargcount" => Some(Object::Int(i64::from(c.kwonly_count))),
@@ -17030,8 +17042,10 @@ pub(crate) fn file_write(args: &[Object]) -> Result<Object, RuntimeError> {
             }
             // Text writes commit fully and report the *character* count
             // (CPython `TextIOWrapper.write`), never a partial byte tally.
-            f.write_text_all(&f.encode_text(s)?)?;
-            s.chars().count()
+            if !f.mem_text_append(s) {
+                f.write_text_all(&f.encode_text(s)?)?;
+            }
+            crate::object::str_char_len(s)
         }
         // A surrogate-bearing `str`. For an in-memory `StringIO` the lone
         // surrogates ride through the PUA bridge so they round-trip; a real
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index f6585ef5..5ef30475 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3555,10 +3555,17 @@ unsafe extern "C" fn h_stack_attr(
             Object::Type(cls) if Rc::strong_count(cls) > 1 => type_attr(code, ext, cls, pc, arg),
             Object::Module(m) => Interpreter::core_module_attr(code, m, pc, arg)
                 .or_else(|| Interpreter::leaf_fused_local_attr(code, recv, pc, arg)),
-            Object::Code(_) | Object::Frame(_) => code
-                .names
-                .get(arg as usize)
-                .and_then(|name| Interpreter::frame_plain_attr(recv, name, None)),
+            Object::Code(_) | Object::Frame(_) => {
+                let Some(v) = code
+                    .names
+                    .get(arg as usize)
+                    .and_then(|name| Interpreter::frame_plain_attr(recv, name, None))
+                else {
+                    return 1;
+                };
+                crate::drop_hot(std::mem::replace(&mut *at, v));
+                return 0;
+            }
             _ => None,
         };
         let Some(v) = v else {
@@ -3631,10 +3638,19 @@ unsafe extern "C" fn h_local_attr(
             Object::Type(cls) => type_attr(code, ext, cls, pc, arg),
             // A frame's or code object's plain field (a live frame's line
             // needs the full lane, which makes its `lasti` current).
-            Object::Code(_) | Object::Frame(_) => code
-                .names
-                .get(arg as usize)
-                .and_then(|name| Interpreter::frame_plain_attr(recv, name, None)),
+            Object::Code(_) | Object::Frame(_) => {
+                return match code
+                    .names
+                    .get(arg as usize)
+                    .and_then(|name| Interpreter::frame_plain_attr(recv, name, None))
+                {
+                    Some(v) => {
+                        dst.write(v);
+                        0
+                    }
+                    None => 1,
+                };
+            }
             _ => None,
         };
         match v {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 162cc320..5430104e 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -5546,14 +5546,31 @@ impl Interpreter {
     /// nothing may stay pinned until reuse.
     fn recycle_frame_shell(&self, mut shell: Rc<crate::object::FrameShell>) {
         const FRAME_SHELL_POOL_CAP: usize = 64;
-        if shell.is_gen
-            || shell
-                .has_materialized
-                .load(std::sync::atomic::Ordering::Relaxed)
-            || self.frame_shell_pool.borrow().len() >= FRAME_SHELL_POOL_CAP
-        {
+        if shell.is_gen || self.frame_shell_pool.borrow().len() >= FRAME_SHELL_POOL_CAP {
             return;
         }
+        if shell
+            .has_materialized
+            .load(std::sync::atomic::Ordering::Relaxed)
+        {
+            // A frame object nothing but the shell holds (a `sys._getframe`
+            // handle since dropped) dies with the activation, as CPython's
+            // does; one anything else holds keeps the shell too.
+            let Some(m) = Rc::get_mut(&mut shell) else {
+                return;
+            };
+            if !m
+                .materialized
+                .get_mut()
+                .as_ref()
+                .is_some_and(|py| Rc::strong_count(py) == 1)
+            {
+                return;
+            }
+            let py = m.materialized.get_mut().take();
+            *m.has_materialized.get_mut() = false;
+            drop(py);
+        }
         if let Some(m) = Rc::get_mut(&mut shell) {
             m.code.clear();
             m.locals.clear();
@@ -71398,6 +71415,9 @@ struct CodeConstObjects {
     /// generator's `gi_name` and `gi_qualname` while its function's names
     /// are untouched.
     gen_names: std::sync::OnceLock<(Object, Object)>,
+    /// `co_name`, `co_qualname` and `co_filename` as the str objects every
+    /// read returns (CPython stores them; logging reads two per record).
+    ident_strs: std::sync::OnceLock<[Object; 3]>,
     /// The code's native form for the core loop (see [`frame_jit`]).
     #[cfg(feature = "jit")]
     frame_jit: frame_jit::Slot,
@@ -73564,6 +73584,7 @@ fn code_vm_ext_build(code: &CodeObject) -> &CodeConstObjects {
             calls: std::sync::atomic::AtomicU8::new(0),
             name_memos: std::sync::OnceLock::new(),
             gen_names: std::sync::OnceLock::new(),
+            ident_strs: std::sync::OnceLock::new(),
             #[cfg(feature = "jit")]
             frame_jit: frame_jit::Slot::default(),
             cold_sites: std::sync::atomic::AtomicU32::new(0),
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 2e7b44e8..4dc9db62 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -6746,6 +6746,46 @@ impl PyFile {
     /// rewritten to the configured line ending. Universal mode maps `\n`→
     /// `os.linesep` (a no-op on WeavePy's Unix targets); `''`/`'\n'` are
     /// pass-through.
+    /// `StringIO.write(s)` at the end of the buffer with nothing to
+    /// translate or encode (the common logging-handler shape): appends `s`
+    /// and returns `true`. `false` touches nothing; the general path then
+    /// runs (overwrites, newline translation, an explicit codec).
+    pub fn mem_text_append(&self, s: &str) -> bool {
+        if self.encoding.borrow().is_some()
+            || self
+                .errors
+                .borrow()
+                .as_deref()
+                .is_some_and(|e| e != "strict")
+        {
+            return false;
+        }
+        let collapse = match self.newline.borrow().as_deref() {
+            Some("\n" | "") => false,
+            None if self.io_kind.get() == IoKind::StringIO => true,
+            _ => return false,
+        };
+        if collapse && s.as_bytes().contains(&b'\r') {
+            return false;
+        }
+        let Ok(mut backend) = self.backend.try_borrow_mut() else {
+            return false;
+        };
+        let FileBackend::MemText { data, pos } = &mut *backend else {
+            return false;
+        };
+        if *pos != data.len() {
+            return false;
+        }
+        data.push_str(s);
+        *pos = data.len();
+        drop(backend);
+        if collapse {
+            self.record_seen_newlines(s);
+        }
+        true
+    }
+
     fn translate_newlines_write(&self, s: &str) -> String {
         match self.newline.borrow().as_deref() {
             Some("\r") => s.replace('\n', "\r"),

From 078cce6064c871f726d148a90ca78b85f8f52995 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 05:51:19 -0700
Subject: [PATCH 043/173] perf: let warnings.warn read lean frames and skip
 unused instances

- setup_context walks the running interpreter's whole stack (the
  pending lean activations, then the spine) instead of pushing a shell
  for every lean caller first, and reads only the frame stack handle.
  warn runs in the core loop's builtin lane accordingly.
- A built-in category's warning instance is created only when the
  pipeline uses it (an error or a shown warning); an ignored or
  already-issued warning never builds one. A user category's
  constructor still runs first, as in CPython.
- The filename comes from the code object's cached co_filename, and the
  per-call dict probes ("version", "__warningregistry__", "__name__",
  "warnings") reuse key strings with memoized hashes.

Measured (instructions per warn from a call chain, "ignore" filter,
pinned seed): warn("x", DeprecationWarning) 18.3k -> 10.6k (CPython
5.8k); warn("x", UserWarning, stacklevel=2) 18.2k -> 13.3k (CPython
6.7k). Suite geomean 1.0003x vs the previous commit (no workload warns;
code layout noise).
---
 crates/weavepy-vm/src/lib.rs                 |  36 +++++
 crates/weavepy-vm/src/stdlib/warnings_mod.rs | 152 +++++++++++++------
 tests/regrtest/test_warnings_stack_lean.py   |  77 ++++++++++
 3 files changed, 216 insertions(+), 49 deletions(-)
 create mode 100644 tests/regrtest/test_warnings_stack_lean.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 5430104e..b3799f27 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -10587,6 +10587,40 @@ impl Interpreter {
         0
     }
 
+    /// How many activations the whole stack holds (see
+    /// [`Self::flush_pending_through`]).
+    pub(crate) fn stack_depth(&self) -> usize {
+        self.lean_pending.len() + self.frame_stack.borrow().len()
+    }
+
+    /// The code, current line and globals of the activation `depth` levels
+    /// up the whole stack (see [`Self::flush_pending_through`]), without
+    /// pushing any shell. A pending activation is suspended in a call, its
+    /// `pc` past it.
+    pub(crate) fn frame_context_at(
+        &self,
+        depth: usize,
+    ) -> Option<(Rc<CodeObject>, u32, Rc<RefCell<DictData>>)> {
+        let n = self.lean_pending.len();
+        if depth < n {
+            // SAFETY: as in `frame_globals_at`.
+            let frame = unsafe {
+                let act = &*(self.lean_pending[n - 1 - depth] as *const LeanAct);
+                &*act.frame
+            };
+            let lasti = frame.pc.saturating_sub(1) as usize;
+            let line = frame.code.linetable.get(lasti).copied().unwrap_or(0);
+            return Some((frame.code.clone(), line, frame.globals.clone()));
+        }
+        let stack = self.frame_stack.borrow();
+        let shell = stack.get(stack.len().checked_sub(1 + depth - n)?)?;
+        Some((
+            (*shell.code).clone(),
+            shell.current_lineno(),
+            (*shell.globals).clone(),
+        ))
+    }
+
     /// The globals of the activation `depth` levels up the whole stack
     /// (see [`Self::flush_pending_through`]), without pushing any shell.
     pub(crate) fn frame_globals_at(&self, depth: usize) -> Option<Rc<RefCell<DictData>>> {
@@ -71152,6 +71186,8 @@ fn builtin_fn_lane_ok(name: &str) -> bool {
             | "insort_left" | "insort_right"
             // sys
             | "_getframe" | "_getframemodulename"
+            // warnings (which reads the whole stack, lean activations included)
+            | "warn"
     )
 }
 
diff --git a/crates/weavepy-vm/src/stdlib/warnings_mod.rs b/crates/weavepy-vm/src/stdlib/warnings_mod.rs
index cde43e70..d2d28975 100644
--- a/crates/weavepy-vm/src/stdlib/warnings_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/warnings_mod.rs
@@ -295,6 +295,20 @@ fn builtin_kw(
 /// named attribute — `get_warnings_attr` in `_warnings.c`. Attribute
 /// misses and import failures both come back as `None`; the caller falls
 /// back to the internal state.
+/// Key strings the warning pipeline probes dicts with on every call, made
+/// once so each probe reuses the string's memoized hash.
+const K_VERSION: usize = 0;
+const K_REGISTRY: usize = 1;
+const K_NAME: usize = 2;
+const K_WARNINGS: usize = 3;
+
+fn wkey(k: usize) -> &'static Object {
+    static KEYS: std::sync::OnceLock<[Object; 4]> = std::sync::OnceLock::new();
+    &KEYS.get_or_init(|| {
+        ["version", "__warningregistry__", "__name__", "warnings"].map(Object::from_static)
+    })[k]
+}
+
 fn get_warnings_attr(attr: &str, try_import: bool) -> Option<Object> {
     let ip = interp().ok()?;
     let module = if try_import {
@@ -303,7 +317,7 @@ fn get_warnings_attr(attr: &str, try_import: bool) -> Option<Object> {
         ip.module_cache()
             .modules
             .borrow()
-            .get(&StrKey("warnings"))
+            .get(&DictKey(wkey(K_WARNINGS).clone()))
             .cloned()?
     };
     match &module {
@@ -505,7 +519,7 @@ fn already_warned(
     should_set: bool,
 ) -> Result<bool, RuntimeError> {
     let version = with_state(|st| st.filters_version);
-    let version_key = DictKey(Object::from_static("version"));
+    let version_key = DictKey(wkey(K_VERSION).clone());
     let stale = match registry.borrow().get(&version_key) {
         Some(Object::Int(v)) => *v != version,
         _ => true,
@@ -648,6 +662,13 @@ fn normalize_module(filename: &Object) -> Object {
     }
 }
 
+/// `category(message)`: a fresh warning instance.
+fn new_warning(category: &Object, message: &Object) -> Result<Object, RuntimeError> {
+    let ip = interp()?;
+    let globals = ip.builtins_dict();
+    ip.call_object_with_globals(category, &[message.clone()], &[], &globals)
+}
+
 fn is_warning_instance(obj: &Object) -> Option<Rc<TypeObject>> {
     if let Object::Instance(inst) = obj {
         let warning = crate::builtin_types::builtin_types().warning.clone();
@@ -710,11 +731,14 @@ fn warn_explicit_locked(
 
     // Normalise: a Warning *instance* supplies both text (str(message))
     // and category (its class); otherwise the message is the text and a
-    // fresh instance is built by calling the category.
-    let (text, message, category_cls) = match is_warning_instance(&message) {
+    // fresh instance is built by calling the category. A built-in
+    // category's instance is built only once something uses it (an
+    // ignored or already-issued warning never does): its construction
+    // runs no code anyone could observe.
+    let (text, instance, category_cls) = match is_warning_instance(&message) {
         Some(cls) => {
             let text = Object::from_str(interp()?.str_object(&message)?);
-            (text, message, cls)
+            (text, Some(message), cls)
         }
         None => {
             let Object::Type(cls) = &category else {
@@ -724,13 +748,18 @@ fn warn_explicit_locked(
                 )));
             };
             let cls = cls.clone();
-            let ip = interp()?;
-            let globals = ip.builtins_dict();
-            let instance =
-                ip.call_object_with_globals(&category, &[message.clone()], &[], &globals)?;
-            (message, instance, cls)
+            if cls.flags.is_builtin {
+                (message, None, cls)
+            } else {
+                let instance = new_warning(&category, &message)?;
+                (message, Some(instance), cls)
+            }
         }
     };
+    let message = || match &instance {
+        Some(m) => Ok(m.clone()),
+        None => new_warning(&category, &text),
+    };
 
     let key = Object::new_tuple_array([
         text.clone(),
@@ -747,7 +776,7 @@ fn warn_explicit_locked(
     let action = action_str(&action_obj).unwrap_or_default();
 
     if action == "error" {
-        return Err(RuntimeError::PyException(PyException::new(message)));
+        return Err(RuntimeError::PyException(PyException::new(message()?)));
     }
     if action == "ignore" {
         return Ok(Object::None);
@@ -799,7 +828,7 @@ fn warn_explicit_locked(
         call_show_warning(
             &category_cls,
             &text,
-            &message,
+            &message()?,
             filename,
             lineno,
             sourceline,
@@ -947,63 +976,88 @@ fn is_internal_filename(filename: &str) -> bool {
 fn setup_context(
     stacklevel: i64,
     skip_file_prefixes: &[String],
-) -> Result<(String, i64, Object, Object), RuntimeError> {
-    crate::builtins::sync_frame_spine();
+) -> Result<(Object, i64, Object, Object), RuntimeError> {
     // Per-thread frame stack, with the interpreter's own as a fallback
     // (shutdown finalizers run `__del__` without re-activating handles —
     // same fallback `sys._getframe` keeps).
-    let frames: Option<crate::object::FrameStack> =
-        match crate::vm_singletons::current_thread_handles() {
-            Some(h) => Some(h.frame_stack.clone()),
-            None => interp().ok().map(|ip| ip.frame_stack.clone()),
-        };
-    // The walk only needs filename / lineno / globals, all present on
-    // the cheap shells — no `PyFrame` materialisation (RFC 0058).
-    let frame: Option<Rc<crate::object::FrameShell>> = frames.and_then(|fs| {
-        let stack = fs.borrow();
-        if stack.is_empty() {
-            return None;
+    let frames: Option<crate::object::FrameStack> = crate::vm_singletons::current_frame_stack()
+        .or_else(|| interp().ok().map(|ip| ip.frame_stack.clone()));
+    // The walk only needs filename / lineno / globals: the running
+    // interpreter's whole stack (lean activations included) answers
+    // without pushing shells or materialising frames (RFC 0058); another
+    // stack's spine is made whole first.
+    let ip = interp().ok();
+    let whole = match (&ip, &frames) {
+        (Some(ip), Some(fs)) if Rc::ptr_eq(&ip.frame_stack, fs) => true,
+        _ => {
+            crate::builtins::sync_frame_spine();
+            false
+        }
+    };
+    type Ctx = (Rc<weavepy_compiler::CodeObject>, u32, Rc<RefCell<DictData>>);
+    let at = |d: usize| -> Option<Ctx> {
+        if whole {
+            return ip.as_ref()?.frame_context_at(d);
         }
-        let is_internal =
-            |f: &Rc<crate::object::FrameShell>| is_internal_filename(&f.code.filename);
-        let to_skip = |f: &Rc<crate::object::FrameShell>| {
-            is_internal(f)
+        let fs = frames.as_ref()?;
+        let stack = fs.borrow();
+        let shell = stack.get(stack.len().checked_sub(1 + d)?)?;
+        Some((
+            (*shell.code).clone(),
+            shell.current_lineno(),
+            (*shell.globals).clone(),
+        ))
+    };
+    let total = match (whole, &ip, &frames) {
+        (true, Some(ip), _) => ip.stack_depth(),
+        (_, _, Some(fs)) => fs.borrow().len(),
+        _ => 0,
+    };
+    let is_internal =
+        |d: usize| at(d).is_some_and(|(code, _, _)| is_internal_filename(&code.filename));
+    let to_skip = |d: usize| {
+        at(d).is_some_and(|(code, _, _)| {
+            is_internal_filename(&code.filename)
                 || skip_file_prefixes
                     .iter()
-                    .any(|p| f.code.filename.starts_with(p.as_str()))
-        };
-        let mut idx: isize = stack.len() as isize - 1;
+                    .any(|p| code.filename.starts_with(p.as_str()))
+        })
+    };
+    let frame = if total == 0 {
+        None
+    } else {
+        let mut d = 0usize;
         let mut level = stacklevel;
-        if level <= 0 || is_internal(&stack[idx as usize]) {
-            while level > 1 && idx >= 0 {
-                idx -= 1;
+        if level <= 0 || is_internal(0) {
+            while level > 1 && d < total {
+                d += 1;
                 level -= 1;
             }
         } else {
-            while level > 1 && idx >= 0 {
+            while level > 1 && d < total {
                 // next_external_frame: hop to the next non-internal,
                 // non-skipped frame.
                 loop {
-                    idx -= 1;
-                    if idx < 0 || !to_skip(&stack[idx as usize]) {
+                    d += 1;
+                    if d >= total || !to_skip(d) {
                         break;
                     }
                 }
                 level -= 1;
             }
         }
-        if idx < 0 {
+        if d >= total {
             None
         } else {
-            Some(stack[idx as usize].clone())
+            at(d)
         }
-    });
+    };
 
     let (globals, filename, lineno) = match frame {
-        Some(f) => {
-            let filename = f.code.filename.to_string();
-            let lineno = i64::from(f.current_lineno());
-            (f.globals.clone(), filename, lineno)
+        Some((code, line, globals)) => {
+            let filename = crate::builtins::code_synthetic_attr(&code, "co_filename")
+                .unwrap_or_else(|| Object::from_str(&*code.filename));
+            (globals, filename, i64::from(line))
         }
         None => {
             // Late-shutdown / no-frame warning: attributed to `<sys>:0`,
@@ -1016,11 +1070,11 @@ fn setup_context(
                     _ => Rc::new(RefCell::new(DictData::default())),
                 }
             };
-            (sysdict, "<sys>".to_owned(), 0)
+            (sysdict, Object::from_static("<sys>"), 0)
         }
     };
 
-    let registry_key = DictKey(Object::from_static("__warningregistry__"));
+    let registry_key = DictKey(wkey(K_REGISTRY).clone());
     let registry = match globals.borrow().get(&registry_key) {
         Some(r) => Some(r.clone()),
         None => None,
@@ -1033,7 +1087,7 @@ fn setup_context(
             fresh
         }
     };
-    let module = match globals.borrow().get(&StrKey("__name__")) {
+    let module = match globals.borrow().get(&DictKey(wkey(K_NAME).clone())) {
         Some(Object::None) | None => Object::from_static("<string>"),
         Some(m) => m.clone(),
     };
@@ -1079,7 +1133,7 @@ fn warn_with_context(
     warn_explicit_core(
         Object::Type(category_cls),
         message,
-        &Object::from_str(filename),
+        &filename,
         lineno,
         Some(module),
         registry,
diff --git a/tests/regrtest/test_warnings_stack_lean.py b/tests/regrtest/test_warnings_stack_lean.py
new file mode 100644
index 00000000..4e63df9d
--- /dev/null
+++ b/tests/regrtest/test_warnings_stack_lean.py
@@ -0,0 +1,77 @@
+"""warnings.warn attributes a warning to the right frame and line for each
+stacklevel while its callers run lean or compiled (hot loops), filters
+ignored warnings without side effects, and still builds the warning
+object a custom category's constructor sees."""
+
+import sys
+import warnings
+
+with open(__file__) as _f:
+    _SRC = _f.read().splitlines()
+
+
+def L(tag):
+    marker = "# " + tag
+    (line,) = [i + 1 for i, s in enumerate(_SRC) if s.endswith(marker)]
+    return line
+
+
+def inner(level):
+    warnings.warn("w%d" % level, UserWarning, level)  # in-warn
+    return 0
+
+
+def mid(level):
+    return inner(level)  # mid-call
+
+
+def outer(level):
+    return mid(level)  # outer-call
+
+
+def collect(n, level):
+    with warnings.catch_warnings(record=True) as got:
+        warnings.simplefilter("always")
+        for i in range(n):
+            outer(level)  # collect-call
+    return {(w.filename == __file__, w.lineno, str(w.message)) for w in got}
+
+
+made = []
+
+
+class Custom(UserWarning):
+    def __init__(self, *args):
+        made.append(args)
+        super().__init__(*args)
+
+
+def ignored(n):
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        for i in range(n):
+            warnings.warn("x", DeprecationWarning)
+            warnings.warn("y", Custom)
+            warnings.warn("z", UserWarning, stacklevel=2)
+    return len(made)
+
+
+def run(n):
+    for level, tag in ((1, "in-warn"), (2, "mid-call"), (3, "outer-call"), (4, "collect-call")):
+        got = collect(n, level)
+        assert got == {(True, L(tag), "w%d" % level)}, (level, got)
+    made.clear()
+    assert ignored(n) == n
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        try:
+            inner(1)
+        except UserWarning as e:
+            assert str(e) == "w1" and type(e) is UserWarning
+        else:
+            raise AssertionError("no error")
+
+
+run(5)
+run(3000)
+run(3000)

From 44d5972b39606ee1aff9f0cbe87e53fdc69ea62b Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 06:32:27 -0700
Subject: [PATCH 044/173] perf: build co_positions once per code object

code.co_positions() built a 4-tuple per code unit on every call;
traceback's StackSummary asks for it once per frame to read one entry.
The tuples are now built once per code object (kept in its VM extension
with the debug-ranges setting they were made under) and each call
returns an iterator over that shared immutable tuple.

Also: a frame object's lasti refresh tests only the line override (a
pending jump always comes with one) instead of borrowing both cells,
and a lazy f_back reads only the frame stack handle of the thread.

Measured (instructions per call, pinned seed): inspect.stack(0) from a
call chain (1 in 50 iterations) 18.1k -> 15.6k per iteration;
traceback.extract_stack(limit=3) 260k -> 255k. Suite geomean 0.9978x
vs the previous commit.
---
 crates/weavepy-vm/src/builtins.rs | 66 ++++++++++++++++++++-----------
 crates/weavepy-vm/src/lib.rs      |  4 ++
 crates/weavepy-vm/src/object.rs   | 16 ++++----
 3 files changed, 53 insertions(+), 33 deletions(-)

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 0dea36d5..98003a7e 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -5020,33 +5020,51 @@ fn code_co_positions(args: &[Object]) -> Result<Object, RuntimeError> {
             .collect();
         return list_iter(items);
     }
-    let cp = c.to_cpython();
     let debug_ranges = crate::vm_singletons::debug_ranges();
-    let col = |v: Option<u32>| {
-        v.filter(|_| debug_ranges)
-            .map_or(Object::None, |x| Object::Int(i64::from(x)))
+    let build = || -> Vec<Object> {
+        let cp = c.to_cpython();
+        let col = |v: Option<u32>| {
+            v.filter(|_| debug_ranges)
+                .map_or(Object::None, |x| Object::Int(i64::from(x)))
+        };
+        let line = |v: i32| {
+            // -1 marks NO_LOCATION; 0 is a real line (module RESUME).
+            if v < 0 {
+                Object::None
+            } else {
+                Object::Int(i64::from(v))
+            }
+        };
+        cp.positions
+            .iter()
+            .map(|p| {
+                // A NO_LOCATION unit (lineno 0) reports all-None (PEP 657).
+                // With debug ranges disabled only start lines survive, so
+                // end_line collapses onto line (CPython stores no
+                // end-position table; test_endline_and_columntable_none_…).
+                let end_lineno = if debug_ranges { p.end_lineno } else { p.lineno };
+                Object::new_tuple_array([
+                    line(p.lineno),
+                    line(end_lineno),
+                    col(p.col),
+                    col(p.end_col),
+                ])
+            })
+            .collect()
     };
-    let line = |v: i32| {
-        // -1 marks NO_LOCATION; 0 is a real line (module RESUME).
-        if v < 0 {
-            Object::None
-        } else {
-            Object::Int(i64::from(v))
+    // The tuples are built once per code object (`traceback` asks for
+    // every frame's, then reads one entry): an iterator over one shared
+    // immutable tuple of them.
+    if let Some(ext) = crate::code_vm_ext(&c) {
+        let (ranges, all) = ext
+            .positions
+            .get_or_init(|| (debug_ranges, Object::new_tuple(build())));
+        if *ranges == debug_ranges {
+            let it = all.make_iter()?;
+            return Ok(Object::Iter(Rc::new(RefCell::new(it))));
         }
-    };
-    let items = cp
-        .positions
-        .iter()
-        .map(|p| {
-            // A NO_LOCATION unit (lineno 0) reports all-None (PEP 657).
-            // With debug ranges disabled only start lines survive, so
-            // end_line collapses onto line (CPython stores no
-            // end-position table; test_endline_and_columntable_none_…).
-            let end_lineno = if debug_ranges { p.end_lineno } else { p.lineno };
-            Object::new_tuple_array([line(p.lineno), line(end_lineno), col(p.col), col(p.end_col)])
-        })
-        .collect();
-    list_iter(items)
+    }
+    list_iter(build())
 }
 
 /// Wrap a vector of objects as a single-use iterator, mirroring the
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index b3799f27..3974c591 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -71454,6 +71454,9 @@ struct CodeConstObjects {
     /// `co_name`, `co_qualname` and `co_filename` as the str objects every
     /// read returns (CPython stores them; logging reads two per record).
     ident_strs: std::sync::OnceLock<[Object; 3]>,
+    /// `co_positions()` entries, as a tuple of 4-tuples, and the
+    /// debug-ranges setting they were built under.
+    positions: std::sync::OnceLock<(bool, Object)>,
     /// The code's native form for the core loop (see [`frame_jit`]).
     #[cfg(feature = "jit")]
     frame_jit: frame_jit::Slot,
@@ -73621,6 +73624,7 @@ fn code_vm_ext_build(code: &CodeObject) -> &CodeConstObjects {
             name_memos: std::sync::OnceLock::new(),
             gen_names: std::sync::OnceLock::new(),
             ident_strs: std::sync::OnceLock::new(),
+            positions: std::sync::OnceLock::new(),
             #[cfg(feature = "jit")]
             frame_jit: frame_jit::Slot::default(),
             cold_sites: std::sync::atomic::AtomicU32::new(0),
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 4dc9db62..0b0f38b1 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -658,10 +658,8 @@ impl PyFrame {
     /// line jump or `f_lineno` override owns the value.
     pub fn lasti_now(&self) -> u32 {
         let live = self.live.load(std::sync::atomic::Ordering::Relaxed);
-        if !live.is_null()
-            && self.override_lineno.get().is_none()
-            && self.pending_jump.try_borrow().is_ok_and(|j| j.is_none())
-        {
+        // (A pending jump always comes with its line override.)
+        if !live.is_null() && self.override_lineno.get().is_none() {
             // SAFETY: a linked shell is alive: the link is cleared when the
             // shell leaves its spine and again when it is dropped.
             let at = unsafe { (*live).lasti.load(std::sync::atomic::Ordering::Relaxed) };
@@ -1265,7 +1263,8 @@ impl FrameShell {
     /// dispatch loops sync only the shell (see [`PyFrame::live`]). A trace
     /// function's pending line jump owns the value and is left alone.
     pub fn refresh_materialized(&self, py: &PyFrame) {
-        if py.pending_jump.borrow().is_none() && py.override_lineno.get().is_none() {
+        // (A pending jump always comes with its line override.)
+        if py.override_lineno.get().is_none() {
             py.lasti
                 .set(self.lasti.load(std::sync::atomic::Ordering::Relaxed));
         }
@@ -1596,14 +1595,13 @@ impl PyTraceback {
 /// the one it already has, else materialized where it sits on this
 /// thread's spine (linking `f_back` like any other materialization).
 pub(crate) fn materialize_shell(shell: &Rc<FrameShell>) -> Rc<PyFrame> {
-    if let Some(handles) = crate::vm_singletons::current_thread_handles() {
-        let idx = handles
-            .frame_stack
+    if let Some(frame_stack) = crate::vm_singletons::current_frame_stack() {
+        let idx = frame_stack
             .borrow()
             .iter()
             .rposition(|s| Rc::ptr_eq(s, shell));
         if let Some(idx) = idx {
-            if let Some(py) = materialize_stack_at_lazy(&handles.frame_stack, idx) {
+            if let Some(py) = materialize_stack_at_lazy(&frame_stack, idx) {
                 return py;
             }
         }

From 4fff0ea752c15451be97ca1365f55d4ac3190e5f Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 05:18:39 -0700
Subject: [PATCH 045/173] perf: box the tier-2 cache's per-code entries

Every code object that runs gets an entry in the tier-2 cache, and an
entry is about 190 bytes, so the hash table's buckets (spare ones
included) took that much each: after `import asyncio` the table was one
386 KB allocation. Boxing the entries makes a bucket 16 bytes and leaves
only the live entries at full size.

Peak RSS of `-c "import asyncio"` drops by about 0.3 MB; instructions are
unchanged within noise.
---
 crates/weavepy-vm/src/tier2.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index 23df33d4..7622b1e5 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -623,7 +623,7 @@ const CACHE_SWEEP_MIN: usize = 1024;
 /// cache that has grown to `sweep_at` entries first drops those of code
 /// objects that have died (see [`JitState::sweep_at`]).
 fn cache_entry<'a>(
-    cache: &'a mut CodeMap<CacheEntry>,
+    cache: &'a mut CodeMap<Box<CacheEntry>>,
     sweep_at: &mut usize,
     code: &Rc<CodeObject>,
 ) -> &'a mut CacheEntry {
@@ -633,7 +633,7 @@ fn cache_entry<'a>(
     }
     cache
         .entry(Rc::as_ptr(code).cast::<CodeObject>())
-        .or_insert_with(|| CacheEntry::new(code))
+        .or_insert_with(|| Box::new(CacheEntry::new(code)))
 }
 
 /// Give up on OSR for a code object after this many failed validations.
@@ -800,7 +800,10 @@ struct JitState {
     threshold: u32,
     range_budget: bool,
     engine: Option<JitEngine>,
-    cache: CodeMap<CacheEntry>,
+    /// Each code object's entry, boxed: every code object that runs gets
+    /// one, and an entry in line made each of the table's buckets (spare
+    /// ones included) as large as an entry.
+    cache: CodeMap<Box<CacheEntry>>,
     /// The cache size at which [`cache_entry`] next sweeps out the
     /// entries of dead code objects (twice the live count the last sweep
     /// left, so sweeping stays amortized constant time per entry).

From e30c0628a52605cfa873635524f8d80ba4bf62df Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 05:18:46 -0700
Subject: [PATCH 046/173] perf: read cached directory listings in the stdlib
 shadow scan

Every frozen import first checks the `sys.path` entries ahead of the
stdlib for a module that shadows it, and did so with three `stat` calls
per entry (the `os.py` landmark, the package `__init__.py` and the
module file); under `-c` that is the current directory for each of the
hundred or so modules `import asyncio` loads. The scan now probes the
cached directory listings that `find_source` already uses: one `stat`
of the directory per lookup, with the listing's exact spelling as the
case check. The landmark keeps a `stat` for any entry that matches
`os.py` in some case, so a case-insensitive filesystem still finds it.

The `stat` calls were about 2% of a re-import loop's samples.
---
 crates/weavepy-vm/src/import.rs | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/crates/weavepy-vm/src/import.rs b/crates/weavepy-vm/src/import.rs
index e53d928a..d3160e9d 100644
--- a/crates/weavepy-vm/src/import.rs
+++ b/crates/weavepy-vm/src/import.rs
@@ -518,6 +518,10 @@ impl ModuleCache {
     /// is the `pythonXY.zip` landmark or any directory that is itself a
     /// stdlib tree (contains `os.py` — the landmark CPython's `getpath`
     /// uses), e.g. a vendored `Lib/`; scanning stops there.
+    ///
+    /// Every frozen import runs this scan, so it reads the cached directory
+    /// listings as [`Self::find_source`] does (one `stat` per directory
+    /// instead of one per candidate).
     pub fn find_source_shadowing_stdlib(&self, full_name: &str) -> Option<(PathBuf, bool)> {
         let rel: PathBuf = full_name.split('.').collect();
         for dir in self.search_dirs() {
@@ -527,22 +531,33 @@ impl ModuleCache {
             {
                 return None;
             }
-            if dir.join("os.py").is_file() {
+            if Self::has_stdlib_landmark(&dir) {
                 return None;
             }
             // Package directory before module file, as in `find_source`.
-            let pkg_init = dir.join(&rel).join("__init__.py");
-            if pkg_init.is_file() && Self::entry_case_ok(pkg_init.parent().unwrap_or(&pkg_init)) {
-                return Some((pkg_init, true));
-            }
-            let module_file = dir.join(&rel).with_extension("py");
-            if module_file.is_file() && Self::entry_case_ok(&module_file) {
-                return Some((module_file, false));
+            if let Some(hit) = Self::probe_source(&dir, &rel) {
+                return Some(hit);
             }
         }
         None
     }
 
+    /// Whether `dir` holds an `os.py` file, the stdlib landmark. The
+    /// listing settles the usual miss; a name matching in any case is
+    /// confirmed with a `stat`, which also matches a differently cased
+    /// entry where the filesystem ignores case.
+    fn has_stdlib_landmark(dir: &Path) -> bool {
+        const LANDMARK: &str = "os.py";
+        let Some(names) = dir_listing(dir) else {
+            return false;
+        };
+        let candidate = names.contains(std::ffi::OsStr::new(LANDMARK))
+            || names
+                .iter()
+                .any(|n| n.as_encoded_bytes().eq_ignore_ascii_case(LANDMARK.as_bytes()));
+        candidate && dir.join(LANDMARK).is_file()
+    }
+
     /// Whether a frozen name is a bundled *third-party facade* rather
     /// than stdlib. These are PyPI projects WeavePy freezes as a
     /// convenience (`packaging`, `numpy`, `pytest`, …). In CPython they

From fec8749896c9bf9e55f5f30111ec211a6466b585 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 05:21:19 -0700
Subject: [PATCH 047/173] perf: give the per-site cache tables an entry per
 site

A warm code object's method, stamp, polymorphic-attribute, call, field
and leaf-call tables each held an entry (16 to 64 bytes) for every
instruction, although only attribute, global, subscript, special-method
and call instructions ever read one: about three quarters of every table
went unused, and `import asyncio` kept 1.6 MB of them.

Each table now holds an entry per site. A code object's first table
builds its sites' ranks (a `u16` per instruction) and every table keeps
a handle to them, so a lookup still checks one initialization: it maps
the pc to its rank, and a non-site's rank is past the end of every
table. Sites past the 65,535th run uncached. The tables are reached
through accessors (`method_slot`, `stamp_slot`, ...) and pc-indexed
views (`Sites`), so nothing indexes them by pc directly any more.

Peak RSS of `-c "import asyncio"` drops by about 1.6 MB (to 29.1 MB),
`import pathlib, datetime, urllib.parse, statistics` by 1.0 MB and
`import typing, dataclasses, enum` by 0.6 MB. The suite's instructions
are unchanged (geomean 1.000 against the parent).

`test_site_tables` checks that neighboring sites keep their own caches
in warm code and that code with more sites than the tables index still
reads every name right.
---
 crates/weavepy-vm/src/frame_jit.rs     |  39 +--
 crates/weavepy-vm/src/leaf_plan.rs     |  12 +-
 crates/weavepy-vm/src/leaf_plan/jit.rs |   9 +-
 crates/weavepy-vm/src/lib.rs           | 429 ++++++++++++++++++-------
 crates/weavepy-vm/src/tier2.rs         |   6 +-
 tests/regrtest/test_site_tables.py     | 127 ++++++++
 6 files changed, 457 insertions(+), 165 deletions(-)
 create mode 100644 tests/regrtest/test_site_tables.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 5ef30475..4ed0cb96 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -2988,7 +2988,7 @@ unsafe extern "C" fn h_load_global(
         if frame.builtins_obj().is_some() {
             return DECLINED;
         }
-        let Some(slot) = ext.stamp_slots.get().and_then(|s| s.get(pc)) else {
+        let Some(slot) = ext.stamp_slot(pc) else {
             return DECLINED;
         };
         let (gdict, bdict) = (frame.globals.as_ptr(), frame.builtins.as_ptr());
@@ -3083,9 +3083,7 @@ unsafe extern "C" fn h_load_global(
                     && crate::simple_args_prefix(ins, pc + 2) =>
             {
                 let fp = ext
-                    .method_slots
-                    .get()
-                    .and_then(|s| s.get(pc + 1))
+                    .method_slot(pc + 1)
                     .and_then(|ms| ms.peek_unbound(cls.attr_version.get()))
                     .filter(|&fp| crate::fn_is_pure_leaf(&*fp));
                 if let Some(fp) = fp {
@@ -3106,9 +3104,7 @@ unsafe extern "C" fn h_load_global(
             // the `LOAD_ATTR` site's stamp.
             Object::Type(cls) if op_at(pc + 1) == Some(OpCode::LoadAttr) => {
                 if let Some(c) = ext
-                    .stamp_slots
-                    .get()
-                    .and_then(|s| s.get(pc + 1))
+                    .stamp_slot(pc + 1)
                     .and_then(|s| crate::class_attr_hit(s, cls))
                 {
                     return fused(st, c, pc + 1);
@@ -3261,7 +3257,7 @@ unsafe extern "C" fn h_load_method(
                 }
             }
         }
-        let Some(ms) = ext.method_slots.get().and_then(|s| s.get(pc)) else {
+        let Some(ms) = ext.method_slot(pc) else {
             return 1;
         };
         let f = match &*top {
@@ -3537,7 +3533,7 @@ unsafe extern "C" fn h_stack_attr(
                                 Some((k, v)) if crate::slot_name_matches(code, arg, k) => {
                                     crate::field_slot_note(
                                         ext,
-                                        code.instructions.len(),
+                                        code,
                                         pc,
                                         inst,
                                         key_idx,
@@ -3590,9 +3586,7 @@ fn type_attr(
     arg: u32,
 ) -> Option<Object> {
     if let Some(v) = ext
-        .stamp_slots
-        .get()
-        .and_then(|s| s.get(pc))
+        .stamp_slot(pc)
         .and_then(|s| crate::class_attr_hit(s, cls))
     {
         return Some(v);
@@ -4117,8 +4111,7 @@ unsafe extern "C" fn h_property(
             return attr_lane(st, pc, slot, recv, local);
         };
         let Some((getter, gcode)) = crate::code_vm_ext(&*code)
-            .and_then(|ext| ext.method_slots.get())
-            .and_then(|s| s.get(pc))
+            .and_then(|ext| ext.method_slot(pc))
             .and_then(|ms| ms.getter(inst.cls_raw().attr_version.get()))
         else {
             return attr_lane(st, pc, slot, recv, local);
@@ -4698,9 +4691,7 @@ unsafe extern "C" fn h_call(
         // through to them untouched.
         if let Object::Builtin(b) = &*base.add(start) {
             let kind = ext
-                .method_slots
-                .get()
-                .and_then(|s| s.get(pc))
+                .method_slot(pc)
                 .and_then(|s| s.get_leaf(b));
             if matches!(kind, Some(LeafKind::Isinstance | LeafKind::Fast(_)))
                 && Rc::strong_count(b) > 1
@@ -4818,7 +4809,7 @@ unsafe extern "C" fn h_call(
         }
         // A builtin the site has settled as a leaf (one it hasn't runs
         // through the core loop's builtin lane).
-        let Some(slot) = ext.method_slots.get().and_then(|s| s.get(pc)) else {
+        let Some(slot) = ext.method_slot(pc) else {
             return CALL_DECLINED;
         };
         // A builtin the site settled on running in the core loop's lane (a
@@ -4974,9 +4965,7 @@ unsafe extern "C" fn h_call_leaf(
             let start = lenu - argc - 2;
             if let Object::Builtin(b) = &*base.add(start) {
                 let kind = x
-                    .method_slots
-                    .get()
-                    .and_then(|m| m.get(pcu))
+                    .method_slot(pcu)
                     .and_then(|m| m.get_leaf(b));
                 // A container's or string's method with a kernel, on the
                 // receiver in the self slot.
@@ -5837,9 +5826,7 @@ fn compile_with(
     let ninstrs = code.instructions.len();
     // The attribute sites' field caches (allocated here if no read has
     // recorded one yet: the code addresses them).
-    let field_slots: &[FieldSlot] = ext
-        .field_slots
-        .get_or_init(|| (0..ninstrs).map(|_| FieldSlot::empty()).collect());
+    let field_slots = ext.alloc_field_sites(code);
     let mut entries: Vec<bool>;
     let mut globals: Vec<Box<GlobalCache>>;
     let mut slots: Vec<Box<SlotCache>>;
@@ -5995,7 +5982,7 @@ struct Lower<'a> {
     ext: &'a CodeConstObjects,
     nlocals: usize,
     depths: &'a [i64],
-    field_slots: &'a [FieldSlot],
+    field_slots: crate::Sites<'a, FieldSlot>,
     st: Value,
     locals: Value,
     stack: Value,
@@ -6164,7 +6151,7 @@ impl<'a> Lower<'a> {
         ext: &'a CodeConstObjects,
         nlocals: usize,
         depths: &'a [i64],
-        field_slots: &'a [FieldSlot],
+        field_slots: crate::Sites<'a, FieldSlot>,
     ) -> Self {
         let entry = b.create_block();
         b.append_block_params_for_function_params(entry);
diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs
index 71c1ae9d..67bb39e6 100644
--- a/crates/weavepy-vm/src/leaf_plan.rs
+++ b/crates/weavepy-vm/src/leaf_plan.rs
@@ -1086,7 +1086,7 @@ impl Interpreter {
             }};
         }
         let mut consts: &[V] = &plan.consts;
-        let mut stamps: &[crate::StampSlot] = ext.stamp_slots.get().map_or(&[], |s| &s[..]);
+        let mut stamps: crate::Sites<'_, crate::StampSlot> = ext.stamp_sites();
         let mut owned = Owned::new();
         let mut pend = Pending::new();
         let mut ops: &[Op] = &plan.ops;
@@ -1274,7 +1274,7 @@ impl Interpreter {
                         depth += 1;
                         (code, ext, plan, f) = (&**ccode, cext, cplan, callee);
                         consts = &plan.consts;
-                        stamps = ext.stamp_slots.get().map_or(&[], |s| &s[..]);
+                        stamps = ext.stamp_sites();
                         ops = &plan.ops;
                         ip = 0;
                         continue;
@@ -1321,7 +1321,7 @@ impl Interpreter {
                         let c = unsafe { &*callers.get_unchecked(depth).as_ptr() };
                         (ip, code, ext, plan, f) = (c.ip, c.code, c.ext, c.plan, c.f);
                         consts = &plan.consts;
-                        stamps = ext.stamp_slots.get().map_or(&[], |s| &s[..]);
+                        stamps = ext.stamp_sites();
                         ops = &plan.ops;
                         // SAFETY: as above.
                         unsafe {
@@ -1559,7 +1559,7 @@ impl Interpreter {
     fn plan_global(
         &self,
         code: &CodeObject,
-        stamps: &[crate::StampSlot],
+        stamps: crate::Sites<'_, crate::StampSlot>,
         f: &crate::object::PyFunction,
         pc: u16,
     ) -> Option<V> {
@@ -1609,7 +1609,7 @@ impl Interpreter {
         &self,
         code: &CodeObject,
         ext: &CodeConstObjects,
-        stamps: &[crate::StampSlot],
+        stamps: crate::Sites<'_, crate::StampSlot>,
         owned: &mut Owned,
         pend: &Pending,
         src: V,
@@ -1704,7 +1704,7 @@ impl Interpreter {
         if (EFFECT && pend.n > 0) || nest >= NEST {
             return None;
         }
-        let slot = ext.method_slots.get()?.get(pc as usize)?;
+        let slot = ext.method_slot(pc as usize)?;
         let cls = inst.cls_raw();
         // SAFETY: GIL-serialized; nothing here runs Python, so nothing
         // rebinds the getter or its code, or changes the class.
diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs
index c3f7f895..ef6f1365 100644
--- a/crates/weavepy-vm/src/leaf_plan/jit.rs
+++ b/crates/weavepy-vm/src/leaf_plan/jit.rs
@@ -400,8 +400,8 @@ unsafe fn cx<'a>(ctx: *mut Ctx<'static>) -> &'a mut Ctx<'a> {
 
 /// A frame's stamp slots (its global and class attribute caches).
 #[inline(always)]
-fn stamps(ext: &CodeConstObjects) -> &[crate::StampSlot] {
-    ext.stamp_slots.get().map_or(&[], |s| &s[..])
+fn stamps(ext: &CodeConstObjects) -> crate::Sites<'_, crate::StampSlot> {
+    ext.stamp_sites()
 }
 
 unsafe extern "C" fn h_global(
@@ -539,10 +539,7 @@ unsafe extern "C" fn h_field<const EFFECT: bool>(
         let ver = inst.cls_raw().attr_version.get();
         // A property's getter answered (see `plan_getter`): nothing to
         // remember.
-        let site = ext
-            .method_slots
-            .get()
-            .and_then(|s| s.get((pc_name & 0xffff) as usize));
+        let site = ext.method_slot((pc_name & 0xffff) as usize);
         // SAFETY: GIL-serialized; the references don't outlive the check.
         if site.is_some_and(|s| unsafe { s.getter_peek(ver) }.is_some()) {
             return status;
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 3974c591..ef720feb 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -9844,7 +9844,7 @@ impl Interpreter {
         let Some(Object::Instance(inst)) = frame.stack.last() else {
             return None;
         };
-        let slot = code_vm_ext(&frame.code)?.method_slots.get()?.get(pc)?;
+        let slot = code_vm_ext(&frame.code)?.method_slot(pc)?;
         slot.getter(inst.cls_raw().attr_version.get())
     }
 
@@ -11130,9 +11130,7 @@ impl Interpreter {
         depth_cell: *const std::cell::Cell<usize>,
     ) -> Option<(Result<Object, RuntimeError>, usize)> {
         let next = *code.instructions.get(pc + 1)?;
-        let mslots: &[MethodSlot] = ext
-            .and_then(|e| e.method_slots.get())
-            .map_or(&[][..], |s| &s[..]);
+        let mslots: Sites<'_, MethodSlot> = ext.map_or(Sites::empty(), |e| e.method_sites());
         match other {
             Object::Instance(_) | Object::Module(_) => {
                 // A site that verified its pure callee for this class version
@@ -11292,10 +11290,9 @@ impl Interpreter {
             ($cache:ident, $frame:expr, $ext:expr) => {
                 *$cache.get_or_insert_with(|| {
                     if $frame.builtins_obj().is_none() {
-                        $ext.and_then(|e| e.stamp_slots.get())
-                            .map_or(&[][..], |s| &s[..])
+                        $ext.map_or(Sites::empty(), |e| e.stamp_sites())
                     } else {
-                        &[][..]
+                        Sites::empty()
                     }
                 })
             };
@@ -11303,8 +11300,7 @@ impl Interpreter {
         macro_rules! mslots {
             ($cache:ident, $ext:expr) => {
                 *$cache.get_or_insert_with(|| {
-                    $ext.and_then(|e| e.method_slots.get())
-                        .map_or(&[][..], |s| &s[..])
+                    $ext.map_or(Sites::empty(), |e| e.method_sites())
                 })
             };
         }
@@ -11335,8 +11331,8 @@ impl Interpreter {
             // attribute stamps, the method slots) is derived at its first
             // use: every call, return and helper handoff runs this prologue
             // again, and most activations never read it.
-            let mut cold_stamps: Option<&[StampSlot]> = None;
-            let mut cold_mslots: Option<&[MethodSlot]> = None;
+            let mut cold_stamps: Option<Sites<'_, StampSlot>> = None;
+            let mut cold_mslots: Option<Sites<'_, MethodSlot>> = None;
             //
             // The fused local pairs below, one byte per instruction (empty
             // for code that has none).
@@ -14491,7 +14487,7 @@ impl Interpreter {
                             break Some(CoreExit::Helper);
                         }
                         if let Some(e) = ext {
-                            field_slot_note(e, ninstrs, pc, inst, key_idx);
+                            field_slot_note(e, code, pc, inst, key_idx);
                         }
                         let v = Self::clone_operand(v);
                         // SAFETY: the receiver (droppable) is replaced in place.
@@ -17094,7 +17090,10 @@ impl Interpreter {
         let Some(shape) = code_store_init(code) else {
             return false;
         };
-        let Some(slots) = code_vm_ext(code).and_then(|e| e.field_slots.get()) else {
+        let Some(slots) = code_vm_ext(code)
+            .filter(|e| e.field_slots.is_allocated())
+            .map(|e| e.field_slots.view())
+        else {
             return false;
         };
         let ver = ty.attr_version.get();
@@ -19027,9 +19026,7 @@ impl Interpreter {
             return None;
         }
         let fast = code_vm_ext(code)?
-            .method_slots
-            .get()?
-            .get(pc)?
+            .method_slot(pc)?
             .get_native_subscript(cls.attr_version.get(), leaf_builtins::generation())?;
         #[cfg(test)]
         NATIVE_SUBSCRIPT_CACHE_HITS.with(|hits| hits.set(hits.get() + 1));
@@ -19051,7 +19048,7 @@ impl Interpreter {
         recv: &Object,
         attr_pc: usize,
         name_idx: u32,
-        mslots: &[MethodSlot],
+        mslots: Sites<'_, MethodSlot>,
         lbase: *const Object,
         nlocals: usize,
         consts: &[Object],
@@ -19119,7 +19116,7 @@ impl Interpreter {
         code: &CodeObject,
         recv: &Object,
         attr_pc: usize,
-        mslots: &[MethodSlot],
+        mslots: Sites<'_, MethodSlot>,
         lbase: *const Object,
         nlocals: usize,
         consts: &[Object],
@@ -19219,7 +19216,7 @@ impl Interpreter {
         recv: &Object,
         attr_pc: usize,
         name_idx: u32,
-        mslots: &[MethodSlot],
+        mslots: Sites<'_, MethodSlot>,
         lbase: *const Object,
         nlocals: usize,
         consts: &[Object],
@@ -19265,14 +19262,15 @@ impl Interpreter {
         let callee: &Rc<CodeObject> = unsafe { &*f.code.as_ptr() };
         let ver = cls.attr_version.get();
         if nargs + 1 == callee.arg_count as usize {
-            let held = mslots[attr_pc]
+            let held = mslots
+                .get(attr_pc)?
                 .get_held(ver)
                 .filter(|h| std::ptr::eq(Rc::as_ptr(h), fp));
             if let (Some(ext), Some(held)) = (code_vm_ext(code), held) {
                 let func = Rc::downgrade(&held);
                 leaf_site_set(
                     ext,
-                    code.instructions.len(),
+                    code,
                     attr_pc,
                     Some(LeafSiteData {
                         ver,
@@ -19346,7 +19344,7 @@ impl Interpreter {
         let r = self.pure_leaf_call(code_rc, f, &args[..total], effect, depth);
         let Some(v) = r else {
             if let Some(ext) = code_vm_ext(code) {
-                leaf_site_set(ext, code.instructions.len(), attr_pc, None);
+                leaf_site_set(ext, code, attr_pc, None);
             }
             return SiteCall::Missed;
         };
@@ -19786,7 +19784,7 @@ impl Interpreter {
             }
             field_slot_note(
                 ext,
-                code.instructions.len(),
+                code,
                 cache_pc as usize,
                 inst,
                 key_idx,
@@ -21896,7 +21894,7 @@ impl Interpreter {
             }
             // A scalar class attribute the instance doesn't shadow, or a
             // named tuple field (see `leaf_attr_resolve_site`).
-            if let Some(slot) = ext.stamp_slots.get().and_then(|s| s.get(attr_pc)) {
+            if let Some(slot) = ext.stamp_slot(attr_pc) {
                 if let Some(v) = class_attr_hit_via(slot, inst.cls_raw(), CLASS_ATTR_VIA_INSTANCE) {
                     if !inst_may_shadow(inst, code, name_idx) {
                         return Some(v);
@@ -21921,7 +21919,7 @@ impl Interpreter {
                 return None;
             }
             if let Some(ext) = ext {
-                field_slot_note(ext, code.instructions.len(), attr_pc, inst, key_idx);
+                field_slot_note(ext, code, attr_pc, inst, key_idx);
             }
             return Some(Self::clone_operand(v));
         }
@@ -22009,14 +22007,14 @@ impl Interpreter {
                         } if ver == cached => {
                             let hit = indexed(key_idx);
                             if let (Some(_), Some(ext)) = (hit, ext) {
-                                field_slot_note(ext, code.instructions.len(), pc, inst, key_idx);
+                                field_slot_note(ext, code, pc, inst, key_idx);
                             }
                             hit
                         }
                         _ => None,
                     };
                     primary.or_else(|| {
-                        let poly = code_vm_ext(code)?.attr_poly.get()?.get(pc)?;
+                        let poly = code_vm_ext(code)?.attr_poly_slot(pc)?;
                         indexed(poly.index(ver)?)
                     })
                 }
@@ -22073,8 +22071,7 @@ impl Interpreter {
         let cls = inst.cls_raw();
         let ext = code_vm_ext(code);
         if let Some((ver, idx)) = ext
-            .and_then(|e| e.field_slots.get())
-            .and_then(|slots| slots.get(attr_pc))
+            .and_then(|e| e.field_slot(attr_pc))
             .map(FieldSlot::get)
         {
             if cls.attr_version.get() == ver && !crate::capi_watchers::dicts_active() {
@@ -22123,7 +22120,7 @@ impl Interpreter {
                 let old = std::mem::replace(slot, unsafe { std::ptr::read(value) });
                 drop(old);
                 if let Some(ext) = ext {
-                    field_slot_note(ext, code.instructions.len(), attr_pc, inst, key_idx);
+                    field_slot_note(ext, code, attr_pc, inst, key_idx);
                 }
                 true
             }
@@ -22232,7 +22229,7 @@ impl Interpreter {
         attr_pc: usize,
         cursor: &mut Option<usize>,
     ) -> Option<bool> {
-        let (ver, idx) = ext.field_slots.get()?.get(attr_pc)?.get();
+        let (ver, idx) = ext.field_slot(attr_pc)?.get();
         if inst.cls_raw().attr_version.get() != ver || crate::capi_watchers::dicts_active() {
             return None;
         }
@@ -22354,7 +22351,7 @@ impl Interpreter {
                         // SAFETY: a read between two instructions.
                         let pos = unsafe { inst.dict.split_peek() }.and_then(|s| s.position(name));
                         if let Some(pos) = pos.and_then(|p| u32::try_from(p).ok()) {
-                            field_slot_note(ext, code.instructions.len(), attr_pc, inst, pos);
+                            field_slot_note(ext, code, attr_pc, inst, pos);
                         }
                     }
                     return true;
@@ -25110,8 +25107,7 @@ impl Interpreter {
         // Inspect an existing table first. A Python-defined subscript
         // must not allocate a native cache or pay the temporary probe.
         let cached = code_vm_ext(code)
-            .and_then(|ext| ext.method_slots.get())
-            .and_then(|slots| slots.get(cache_pc as usize))
+            .and_then(|ext| ext.method_slot(cache_pc as usize))
             .and_then(|slot| slot.get_native_subscript(version, generation));
         let fast = match cached {
             Some(fast) => {
@@ -25177,8 +25173,7 @@ impl Interpreter {
         let version = cls.attr_version.get();
         let generation = leaf_builtins::generation();
         let cached = code_vm_ext(code)
-            .and_then(|ext| ext.method_slots.get())
-            .and_then(|slots| slots.get(cache_pc as usize))
+            .and_then(|ext| ext.method_slot(cache_pc as usize))
             .and_then(|slot| slot.get_native_subscript(version, generation));
         if cached.is_some() {
             return cached;
@@ -26540,10 +26535,7 @@ impl Interpreter {
         receiver: &Object,
         depth: usize,
     ) -> Option<Object> {
-        let slot = code_vm_ext(code)?
-            .method_slots
-            .get()?
-            .get(cache_pc as usize)?;
+        let slot = code_vm_ext(code)?.method_slot(cache_pc as usize)?;
         let cls = inst.cls_raw();
         // SAFETY: GIL-serialized (the caller declines free threading); the
         // getter evaluation runs no Python, so nothing rebinds either.
@@ -71375,21 +71367,25 @@ struct CodeConstObjects {
     /// before falling back to a byte compare, and `STORE_ATTR` inserts
     /// the pooled key without a per-store pool lookup.
     name_objs: Vec<Object>,
+    /// Each instruction's rank among the code's *sites* (see
+    /// [`site_ranks`]), which index the per-site tables below, or
+    /// [`NO_SITE`]; built with the first of those tables.
+    site_ranks: std::sync::OnceLock<std::sync::Arc<[u16]>>,
     /// RFC 0077 WS4: the resolved method per `LoadAttrMethod` site (see
     /// [`MethodSlot`]). Allocated on the first method-shape hit in this
-    /// code object, one slot per instruction so `cache_pc` indexes it.
-    method_slots: std::sync::OnceLock<Box<[MethodSlot]>>,
+    /// code object.
+    method_slots: SiteSlots<MethodSlot>,
     /// The dict stamps a `LOAD_GLOBAL` site last validated against (see
     /// [`StampSlot`]); allocated on the first global hit in this code
-    /// object, one slot per instruction.
-    stamp_slots: std::sync::OnceLock<Box<[StampSlot]>>,
+    /// object.
+    stamp_slots: SiteSlots<StampSlot>,
     /// Polymorphic instance-attribute entries per `LOAD_ATTR` site (see
     /// [`AttrPoly`]); allocated on the first polymorphic resolution in
-    /// this code object, one slot per instruction.
-    attr_poly: std::sync::OnceLock<Box<[AttrPoly]>>,
+    /// this code object.
+    attr_poly: SiteSlots<AttrPoly>,
     /// Inline-call shapes per `CALL` site (see [`CallSlot`]); allocated
     /// on the first inline call in this code object.
-    call_slots: std::sync::OnceLock<Box<[CallSlot]>>,
+    call_slots: SiteSlots<CallSlot>,
     /// Whether this code is a *pure leaf* (see [`code_is_pure_leaf`]):
     /// `0` not yet decided, `1` no, `2` general leaf, `3` argument return,
     /// `4` constant return, `5` small-int return, `6` attribute return,
@@ -71418,7 +71414,7 @@ struct CodeConstObjects {
     gen_start: std::sync::atomic::AtomicU32,
     /// Split-layout attribute shortcuts per `LOAD_ATTR` site (see
     /// [`FieldSlot`]); allocated on the first recorded one.
-    field_slots: std::sync::OnceLock<Box<[FieldSlot]>>,
+    field_slots: SiteSlots<FieldSlot>,
     /// The `BINARY_OP` and `COMPARE_OP` sites the core loop has run on a
     /// natively served operand (see `stdlib::datetime_native`), for the
     /// frame compiler to send through its helpers instead of the scalar
@@ -71431,7 +71427,7 @@ struct CodeConstObjects {
     instance_sites: std::sync::OnceLock<Box<[std::sync::atomic::AtomicBool]>>,
     /// Verified frameless method calls per method-load site (see
     /// [`LeafSite`]); allocated on the first recorded one.
-    leaf_sites: std::sync::OnceLock<Box<[LeafSite]>>,
+    leaf_sites: SiteSlots<LeafSite>,
     /// A leaf body's translation for the frameless evaluator (see
     /// [`leaf_plan`]), or `None` when the body has none.
     leaf_plan: std::sync::OnceLock<Option<Box<leaf_plan::LeafPlan>>>,
@@ -71473,6 +71469,205 @@ struct CodeConstObjects {
 /// [`CodeConstObjects::gen_start`]'s mark of a body with a `SEND`.
 const GEN_DELEGATES: u32 = 1 << 31;
 
+/// The rank [`site_ranks`] gives an instruction that isn't a site (or one
+/// past the 65,535th): no per-site table has an entry at it.
+const NO_SITE: u16 = u16::MAX;
+
+/// Whether instructions of `op` are *sites*: the instructions whose pcs
+/// the per-site tables of [`CodeConstObjects`] are read and written at.
+/// An access at any other instruction finds no entry, as an unallocated
+/// table does.
+#[inline]
+fn is_site_op(op: OpCode) -> bool {
+    matches!(
+        op,
+        OpCode::LoadGlobal
+            | OpCode::LoadAttr
+            | OpCode::LoadMethodAttr
+            | OpCode::LoadSuperAttr
+            | OpCode::StoreAttr
+            | OpCode::LoadSpecial
+            | OpCode::BinarySubscr
+            | OpCode::StoreSubscr
+            | OpCode::Call
+            | OpCode::CallSelf
+            | OpCode::CallKw
+            | OpCode::CallEx
+    )
+}
+
+/// Each instruction's rank among `code`'s sites (see [`is_site_op`]), in
+/// order, or [`NO_SITE`].
+///
+/// The per-site tables hold an entry per site rather than per
+/// instruction: most instructions (local and constant loads, stores,
+/// jumps) never read one, and entries tens of bytes each for every
+/// instruction made the warm code of a few imports hold megabytes of
+/// tables.
+fn site_ranks(code: &CodeObject) -> std::sync::Arc<[u16]> {
+    let mut next = 0u16;
+    code.instructions
+        .iter()
+        .map(|i| {
+            if is_site_op(i.op) && next < NO_SITE {
+                next += 1;
+                next - 1
+            } else {
+                NO_SITE
+            }
+        })
+        .collect()
+}
+
+/// A per-site table of [`CodeConstObjects`]: an entry per site (see
+/// [`site_ranks`]), allocated on first use. [`Self::get`] and
+/// [`Self::view`] map an instruction's pc to its entry.
+struct SiteSlots<T>(std::sync::OnceLock<SiteTable<T>>);
+
+/// An allocated [`SiteSlots`]: the code's site ranks (shared by its
+/// tables, and kept here so that a lookup checks one initialization) and
+/// the entries.
+struct SiteTable<T> {
+    ranks: std::sync::Arc<[u16]>,
+    slots: Box<[T]>,
+}
+
+impl<T> SiteSlots<T> {
+    const fn new() -> Self {
+        Self(std::sync::OnceLock::new())
+    }
+
+    /// Whether the table is allocated.
+    #[inline]
+    fn is_allocated(&self) -> bool {
+        self.0.get().is_some()
+    }
+
+    /// The entry of the instruction at `pc`, if the table is allocated
+    /// and the instruction is a site.
+    #[inline(always)]
+    fn get(&self, pc: usize) -> Option<&T> {
+        let t = self.0.get()?;
+        // (`NO_SITE` is past the end of every table, which has at most
+        // `NO_SITE` entries.)
+        t.slots.get(usize::from(*t.ranks.get(pc)?))
+    }
+
+    /// The pc-indexed view of the table (empty while unallocated).
+    #[inline(always)]
+    fn view(&self) -> Sites<'_, T> {
+        match self.0.get() {
+            Some(t) => Sites {
+                ranks: &t.ranks,
+                slots: &t.slots,
+            },
+            None => Sites::empty(),
+        }
+    }
+}
+
+/// A view of a per-site table that takes instruction pcs (see
+/// [`SiteSlots`]); empty when the table isn't allocated.
+pub(crate) struct Sites<'a, T> {
+    ranks: &'a [u16],
+    slots: &'a [T],
+}
+
+impl<T> Clone for Sites<'_, T> {
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<T> Copy for Sites<'_, T> {}
+
+impl<'a, T> Sites<'a, T> {
+    /// The view of no table.
+    pub(crate) const fn empty() -> Self {
+        Sites {
+            ranks: &[],
+            slots: &[],
+        }
+    }
+
+    /// The entry of the instruction at `pc`, if it is a site.
+    #[inline(always)]
+    pub(crate) fn get(&self, pc: usize) -> Option<&'a T> {
+        // (`NO_SITE` is past the end of every table, which has at most
+        // `NO_SITE` entries.)
+        self.slots.get(usize::from(*self.ranks.get(pc)?))
+    }
+}
+
+impl CodeConstObjects {
+    /// The view of `table` (one of this object's per-site tables),
+    /// allocated (with an entry per site of `code`, this object's code) if
+    /// it isn't yet.
+    fn alloc_sites<'a, T>(
+        &'a self,
+        code: &CodeObject,
+        table: &'a SiteSlots<T>,
+        empty: impl Fn() -> T,
+    ) -> Sites<'a, T> {
+        table.0.get_or_init(|| {
+            let ranks = self.site_ranks.get_or_init(|| site_ranks(code)).clone();
+            let n = ranks.iter().filter(|&&r| r != NO_SITE).count();
+            SiteTable {
+                ranks,
+                slots: (0..n).map(|_| empty()).collect(),
+            }
+        });
+        table.view()
+    }
+
+    /// The method slot of the instruction at `pc` (see
+    /// [`Self::method_slots`]).
+    #[inline(always)]
+    pub(crate) fn method_slot(&self, pc: usize) -> Option<&MethodSlot> {
+        self.method_slots.get(pc)
+    }
+
+    /// The stamp slot of the instruction at `pc` (see
+    /// [`Self::stamp_slots`]).
+    #[inline(always)]
+    pub(crate) fn stamp_slot(&self, pc: usize) -> Option<&StampSlot> {
+        self.stamp_slots.get(pc)
+    }
+
+    /// The field slot of the instruction at `pc` (see
+    /// [`Self::field_slots`]).
+    #[inline(always)]
+    pub(crate) fn field_slot(&self, pc: usize) -> Option<&FieldSlot> {
+        self.field_slots.get(pc)
+    }
+
+    /// The polymorphic attribute entries of the instruction at `pc` (see
+    /// [`Self::attr_poly`]).
+    #[inline(always)]
+    pub(crate) fn attr_poly_slot(&self, pc: usize) -> Option<&AttrPoly> {
+        self.attr_poly.get(pc)
+    }
+
+    /// The method slots by pc (see [`Self::method_slots`]).
+    #[inline(always)]
+    pub(crate) fn method_sites(&self) -> Sites<'_, MethodSlot> {
+        self.method_slots.view()
+    }
+
+    /// The stamp slots by pc (see [`Self::stamp_slots`]).
+    #[inline(always)]
+    pub(crate) fn stamp_sites(&self) -> Sites<'_, StampSlot> {
+        self.stamp_slots.view()
+    }
+
+    /// The field slots by pc (see [`Self::field_slots`]), allocated if
+    /// they aren't yet; `code` is this object's code.
+    #[cfg(feature = "jit")]
+    pub(crate) fn alloc_field_sites(&self, code: &CodeObject) -> Sites<'_, FieldSlot> {
+        self.alloc_sites(code, &self.field_slots, FieldSlot::empty)
+    }
+}
+
 /// The bound the core loop checks a code object's entry pc against, so it
 /// dispatches each instruction without a bounds check: the instruction
 /// count when no path from an instruction can leave the code, else `0`
@@ -71519,10 +71714,10 @@ fn code_dispatch_len(code: &CodeObject) -> usize {
 }
 
 /// Whether `ext`'s code (of `ninstrs` instructions) may allocate its
-/// per-instruction side tables, after one more site execution found one
+/// per-site side tables, after one more site execution found one
 /// unallocated.
 ///
-/// Each table holds an entry per instruction, tens of bytes each, while
+/// Each table holds an entry per site, tens of bytes each, while
 /// most code runs only a few times (a function an import calls once or
 /// twice): allocating the tables at a site's first run cost a small
 /// program's start-up about 1.5 MB. So the tables wait until the code's
@@ -71540,6 +71735,39 @@ fn site_tables_warm(ext: &CodeConstObjects, ninstrs: usize) -> bool {
     n as usize >= ninstrs
 }
 
+/// The entry of `table` (one of `ext`'s per-site tables) for the
+/// instruction at `pc`, allocating the table once `code` (`ext`'s code) is
+/// warm (see [`site_tables_warm`]).
+#[inline(always)]
+fn warm_site<'a, T>(
+    ext: &'a CodeConstObjects,
+    code: &CodeObject,
+    table: &'a SiteSlots<T>,
+    pc: usize,
+    empty: fn() -> T,
+) -> Option<&'a T> {
+    match table.0.get() {
+        Some(t) => t.slots.get(usize::from(*t.ranks.get(pc)?)),
+        None => warm_site_alloc(ext, code, table, pc, empty),
+    }
+}
+
+/// [`warm_site`] for a table not yet allocated.
+#[cold]
+#[inline(never)]
+fn warm_site_alloc<'a, T>(
+    ext: &'a CodeConstObjects,
+    code: &CodeObject,
+    table: &'a SiteSlots<T>,
+    pc: usize,
+    empty: fn() -> T,
+) -> Option<&'a T> {
+    if !site_tables_warm(ext, code.instructions.len()) {
+        return None;
+    }
+    ext.alloc_sites(code, table, empty).get(pc)
+}
+
 /// Let `code`'s side tables be allocated from now on (see
 /// [`site_tables_warm`]): the code is hot.
 pub(crate) fn mark_site_tables_warm(code: &CodeObject) {
@@ -71690,15 +71918,7 @@ fn builtin_recv_tag(recv: &Object) -> Option<u64> {
 #[inline]
 fn code_call_slot(code: &CodeObject, pc: usize) -> Option<&CallSlot> {
     let ext = code_vm_ext(code)?;
-    let n = code.instructions.len();
-    match ext.call_slots.get() {
-        Some(slots) => slots,
-        None if site_tables_warm(ext, n) => ext
-            .call_slots
-            .get_or_init(|| (0..n).map(|_| CallSlot::empty()).collect()),
-        None => return None,
-    }
-    .get(pc)
+    warm_site(ext, code, &ext.call_slots, pc, CallSlot::empty)
 }
 
 /// A `LOAD_ATTR` site's polymorphic instance-attribute cache: up to
@@ -71758,15 +71978,7 @@ impl AttrPoly {
 #[inline]
 fn code_attr_poly(code: &CodeObject, cache_pc: u32) -> Option<&AttrPoly> {
     let ext = code_vm_ext(code)?;
-    let n = code.instructions.len();
-    match ext.attr_poly.get() {
-        Some(slots) => slots,
-        None if site_tables_warm(ext, n) => ext
-            .attr_poly
-            .get_or_init(|| (0..n).map(|_| AttrPoly::empty()).collect()),
-        None => return None,
-    }
-    .get(cache_pc as usize)
+    warm_site(ext, code, &ext.attr_poly, cache_pc as usize, AttrPoly::empty)
 }
 
 /// The state a `LOAD_GLOBAL` site last proved its cached slot against:
@@ -71819,7 +72031,7 @@ pub(crate) const FIELD_SLOT_IDX: usize = std::mem::offset_of!((u64, u32), 1);
 /// The [`FieldSlot`] of the attribute site at `pc` of `code`, if the code
 /// has any (native code reads it in line).
 pub(crate) fn code_field_slot(code: &CodeObject, pc: usize) -> Option<*const u8> {
-    let slot = code_vm_ext(code)?.field_slots.get()?.get(pc)?;
+    let slot = code_vm_ext(code)?.field_slot(pc)?;
     Some(std::ptr::from_ref(slot).cast())
 }
 
@@ -71950,7 +72162,7 @@ fn leaf_site_hit(
     pc: usize,
     ver: u64,
 ) -> Option<(*const crate::object::PyFunction, *const CodeObject, bool)> {
-    let site = ext.leaf_sites.get()?.get(pc)?;
+    let site = ext.leaf_sites.get(pc)?;
     // SAFETY: GIL-serialized; the borrow ends before any refill.
     match unsafe { &*site.0.get() } {
         Some(d) if d.ver == ver && d.func.strong_count() > 0 => {
@@ -71971,7 +72183,7 @@ pub(crate) fn builtin_method_site(
     attr_pc: usize,
     call_pc: usize,
 ) -> Option<(u64, LeafKind, Rc<crate::object::BuiltinFn>)> {
-    let slots = ext.method_slots.get()?;
+    let slots = ext.method_sites();
     // SAFETY: GIL-serialized; the borrow ends before any refill.
     let (ver, f) = unsafe { &*slots.get(attr_pc)?.0.get() };
     let MethodSlotFn::Builtin(b) = f else {
@@ -71991,7 +72203,7 @@ pub(crate) fn builtin_method_site(
 /// The leaf builtin's kind the `CALL` at `pc` last settled on.
 #[cfg(feature = "jit")]
 pub(crate) fn call_site_leaf(ext: &CodeConstObjects, pc: usize) -> Option<LeafKind> {
-    let slot = ext.method_slots.get()?.get(pc)?;
+    let slot = ext.method_slot(pc)?;
     // SAFETY: GIL-serialized; the borrow ends here.
     match unsafe { &(*slot.0.get()).1 } {
         MethodSlotFn::Leaf(_, kind) => Some(*kind),
@@ -72017,17 +72229,13 @@ pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
     // SAFETY: GIL-serialized; the borrows end before any refill; a live
     // site's code is its live function's.
     let leaf = ext
-        .leaf_sites
-        .get()
-        .and_then(|s| s.get(pc))
+        .leaf_sites.get(pc)
         .and_then(|s| unsafe { &*s.0.get() }.as_ref())
         .is_some_and(|site| {
             site.func.strong_count() > 0 && !(site.effect && code_calls(unsafe { &*site.code }))
         });
     leaf || ext
-        .method_slots
-        .get()
-        .and_then(|s| s.get(pc))
+        .method_slot(pc)
         .is_some_and(|slot| match unsafe { &(*slot.0.get()).1 } {
             MethodSlotFn::Builtin(_) | MethodSlotFn::Leaf(..) => true,
             MethodSlotFn::Py(w) | MethodSlotFn::Static(w) => {
@@ -72043,13 +72251,13 @@ pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
 
 /// Record (or, with `data` `None`, forget) the site at `pc`'s callee.
 #[inline(never)]
-fn leaf_site_set(ext: &CodeConstObjects, ninstrs: usize, pc: usize, data: Option<LeafSiteData>) {
-    if ext.leaf_sites.get().is_none() && (data.is_none() || !site_tables_warm(ext, ninstrs)) {
+fn leaf_site_set(ext: &CodeConstObjects, code: &CodeObject, pc: usize, data: Option<LeafSiteData>) {
+    if !ext.leaf_sites.is_allocated()
+        && (data.is_none() || !site_tables_warm(ext, code.instructions.len()))
+    {
         return;
     }
-    let sites = ext
-        .leaf_sites
-        .get_or_init(|| (0..ninstrs).map(|_| LeafSite::empty()).collect());
+    let sites = ext.alloc_sites(code, &ext.leaf_sites, LeafSite::empty);
     if let Some(site) = sites.get(pc) {
         site.set(data);
     }
@@ -72080,7 +72288,7 @@ unsafe fn field_slot_hit<'a>(
     pc: usize,
     inst: &'a PyInstance,
 ) -> Option<&'a Object> {
-    let (ver, idx) = ext.field_slots.get()?.get(pc)?.get();
+    let (ver, idx) = ext.field_slot(pc)?.get();
     if inst.cls_raw().attr_version.get() != ver {
         return None;
     }
@@ -72093,17 +72301,15 @@ unsafe fn field_slot_hit<'a>(
 /// (whose class passed the site's version check): kept only when that
 /// position is the split layout's, over the class's own names.
 #[inline(never)]
-fn field_slot_note(ext: &CodeConstObjects, ninstrs: usize, pc: usize, inst: &PyInstance, idx: u32) {
+fn field_slot_note(ext: &CodeConstObjects, code: &CodeObject, pc: usize, inst: &PyInstance, idx: u32) {
     // SAFETY: a read with nothing running (the caller's own guarded read).
     if unsafe { inst.split_field(idx as usize) }.is_none() {
         return;
     }
-    if ext.field_slots.get().is_none() && !site_tables_warm(ext, ninstrs) {
+    if !ext.field_slots.is_allocated() && !site_tables_warm(ext, code.instructions.len()) {
         return;
     }
-    let slots = ext
-        .field_slots
-        .get_or_init(|| (0..ninstrs).map(|_| FieldSlot::empty()).collect());
+    let slots = ext.alloc_sites(code, &ext.field_slots, FieldSlot::empty);
     if let Some(slot) = slots.get(pc) {
         slot.set((inst.cls_raw().attr_version.get(), idx));
     }
@@ -72131,10 +72337,7 @@ fn native_site_note(ext: &CodeConstObjects, ninstrs: usize, pc: usize) {
 /// Whether the `LOAD_ATTR` at `pc` has recorded a split-layout field (see
 /// [`FieldSlot`]).
 pub(crate) fn field_site(ext: &CodeConstObjects, pc: usize) -> bool {
-    ext.field_slots
-        .get()
-        .and_then(|s| s.get(pc))
-        .is_some_and(|s| s.get().0 != 0)
+    ext.field_slot(pc).is_some_and(|s| s.get().0 != 0)
 }
 
 /// Whether the operator at `pc` has run on a natively served operand.
@@ -72155,10 +72358,7 @@ fn instance_site_note(ext: &CodeConstObjects, ninstrs: usize, pc: usize) {
 /// method slot holds one).
 #[cfg(feature = "jit")]
 pub(crate) fn property_site(ext: &CodeConstObjects, pc: usize) -> bool {
-    ext.method_slots
-        .get()
-        .and_then(|s| s.get(pc))
-        .is_some_and(MethodSlot::holds_getter)
+    ext.method_slot(pc).is_some_and(MethodSlot::holds_getter)
 }
 
 /// Whether the `BINARY_OP` at `pc` has run on an instance operand.
@@ -72261,15 +72461,7 @@ fn drop_hot(v: Object) {
 #[inline]
 fn code_stamp_slot(code: &CodeObject, cache_pc: u32) -> Option<&StampSlot> {
     let ext = code_vm_ext(code)?;
-    let n = code.instructions.len();
-    match ext.stamp_slots.get() {
-        Some(slots) => slots,
-        None if site_tables_warm(ext, n) => ext
-            .stamp_slots
-            .get_or_init(|| (0..n).map(|_| StampSlot::empty()).collect()),
-        None => return None,
-    }
-    .get(cache_pc as usize)
+    warm_site(ext, code, &ext.stamp_slots, cache_pc as usize, StampSlot::empty)
 }
 
 /// Tags of a `LOAD_ATTR` site's class-attribute stamp (`[attr_version,
@@ -72907,15 +73099,7 @@ impl MethodSlot {
 #[inline]
 fn code_method_slot(code: &CodeObject, cache_pc: u32) -> Option<&MethodSlot> {
     let ext = code_vm_ext(code)?;
-    let n = code.instructions.len();
-    match ext.method_slots.get() {
-        Some(slots) => slots,
-        None if site_tables_warm(ext, n) => ext
-            .method_slots
-            .get_or_init(|| (0..n).map(|_| MethodSlot::empty()).collect()),
-        None => return None,
-    }
-    .get(cache_pc as usize)
+    warm_site(ext, code, &ext.method_slots, cache_pc as usize, MethodSlot::empty)
 }
 
 /// The plain function class `cls` (at attribute version `ver`) resolves
@@ -73604,19 +73788,20 @@ fn code_vm_ext_build(code: &CodeObject) -> &CodeConstObjects {
                 .iter()
                 .map(|n| crate::stdlib::sys::intern_name(n))
                 .collect(),
-            method_slots: std::sync::OnceLock::new(),
-            stamp_slots: std::sync::OnceLock::new(),
-            attr_poly: std::sync::OnceLock::new(),
-            call_slots: std::sync::OnceLock::new(),
+            site_ranks: std::sync::OnceLock::new(),
+            method_slots: SiteSlots::new(),
+            stamp_slots: SiteSlots::new(),
+            attr_poly: SiteSlots::new(),
+            call_slots: SiteSlots::new(),
             pure_leaf: std::sync::atomic::AtomicU8::new(0),
             leaf_tries: std::sync::atomic::AtomicU64::new(0),
             fast_pairs: std::sync::OnceLock::new(),
             gen_fast: std::sync::atomic::AtomicU8::new(0),
             gen_start: std::sync::atomic::AtomicU32::new(0),
-            field_slots: std::sync::OnceLock::new(),
+            field_slots: SiteSlots::new(),
             native_sites: std::sync::OnceLock::new(),
             instance_sites: std::sync::OnceLock::new(),
-            leaf_sites: std::sync::OnceLock::new(),
+            leaf_sites: SiteSlots::new(),
             leaf_plan: std::sync::OnceLock::new(),
             returns_none: std::sync::atomic::AtomicU8::new(0),
             store_init: std::sync::OnceLock::new(),
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index 7622b1e5..0dbafcbb 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -10924,11 +10924,7 @@ unsafe fn cached_chain_peek<'a>(
                         .or_else(|| slots.get(name.as_ref()))
                 }
                 _ => {
-                    let index = extension
-                        .attr_poly
-                        .get()?
-                        .get(pc as usize)?
-                        .index(version)?;
+                    let index = extension.attr_poly_slot(pc as usize)?.index(version)?;
                     // SAFETY: the same callback-free, rooted interval.
                     let (key, value) = unsafe { inst.attr_peek_index(index as usize) }?;
                     key_is(key, name).then_some(value)
diff --git a/tests/regrtest/test_site_tables.py b/tests/regrtest/test_site_tables.py
new file mode 100644
index 00000000..d7220a99
--- /dev/null
+++ b/tests/regrtest/test_site_tables.py
@@ -0,0 +1,127 @@
+"""Per-site caches must stay with their own instruction.
+
+The interpreter keeps its attribute, global, method and call caches in
+tables with an entry per *site* instruction rather than per instruction,
+so each lookup maps the instruction to its entry. These checks run warm
+code whose neighboring sites read different names from the same objects
+(a cache answered for the wrong site returns the other name's value), and
+code with more sites than the tables index.
+"""
+
+
+class A:
+    __slots__ = ()
+    k = "A.k"
+
+    def __init__(self):
+        pass
+
+
+class P:
+    def __init__(self, x, y, z):
+        self.x = x
+        self.y = y
+        self.z = z
+
+    def total(self):
+        return self.x + self.y + self.z
+
+    def swap(self):
+        return P(self.y, self.x, self.z)
+
+
+class Q(P):
+    def total(self):
+        return -(self.x + self.y + self.z)
+
+
+class R:
+    def __init__(self, x, y, z):
+        self.z = z
+        self.y = y
+        self.x = x
+
+    def total(self):
+        return self.x * 100 + self.y * 10 + self.z
+
+
+SCALE = 3
+
+
+def neighbors(objs):
+    out = []
+    for o in objs:
+        # Adjacent attribute sites with different names on one receiver,
+        # separated by non-site instructions.
+        a = o.x
+        b = o.y
+        c = o.z
+        out.append((a, b, c, o.total(), SCALE, len(out)))
+    return out
+
+
+class Ctx:
+    def __init__(self, log, tag):
+        self.log = log
+        self.tag = tag
+
+    def __enter__(self):
+        self.log.append("enter " + self.tag)
+        return self.tag
+
+    def __exit__(self, *exc):
+        self.log.append("exit " + self.tag)
+        return False
+
+
+class Ctx2(Ctx):
+    def __enter__(self):
+        self.log.append("enter2 " + self.tag)
+        return self.tag * 2
+
+
+def contexts(n):
+    log = []
+    got = []
+    for i in range(n):
+        with Ctx(log, "a") as x:
+            got.append(x)
+        with (Ctx2 if i % 2 else Ctx)(log, "b") as y:
+            got.append(y)
+    return log, got
+
+
+for round_ in range(40):
+    objs = [P(1, 2, 3), Q(4, 5, 6), R(7, 8, 9), P(10, 20, 30)]
+    if round_ == 20:
+        SCALE = 4
+    got = neighbors(objs)
+    scale = 3 if round_ < 20 else 4
+    assert got == [
+        (1, 2, 3, 6, scale, 0),
+        (4, 5, 6, -15, scale, 1),
+        (7, 8, 9, 789, scale, 2),
+        (10, 20, 30, 60, scale, 3),
+    ], (round_, got)
+    swapped = [o.swap() for o in objs if type(o) is P]
+    assert [(s.x, s.y, s.z) for s in swapped] == [(2, 1, 3), (20, 10, 30)]
+    assert A().k == "A.k"
+
+log, got = None, None
+for _ in range(20):
+    log, got = contexts(6)
+assert got == ["a", "b", "a", "bb"] * 3, got
+assert log[:4] == ["enter a", "exit a", "enter b", "exit b"], log[:4]
+assert log[6:8] == ["enter2 b", "exit b"], log[6:8]
+
+# More attribute sites than the tables index (65,535): the ones past
+# the limit run uncached, and every site still reads its own name.
+lines = ["def many(o):", "    t = 0"]
+for i in range(13200):
+    lines.append("    t += o.x - o.z + o.y - o.y + o.z")
+lines.append("    return t, o.y, o.x")
+ns = {}
+exec("\n".join(lines), ns)
+many = ns["many"]
+for o in (P(5, 7, 2), R(5, 7, 2), Q(5, 7, 2)) * 2:
+    assert many(o) == (13200 * 5, 7, 5), many(o)

From 8c0f34e6a48a401feb8ea49fb83bd99df9fd0715 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 06:11:44 -0700
Subject: [PATCH 048/173] perf: build the tier-2 engine at the first compile,
 not at start-up

Start-up ended by building the tier-2 engine and compiling a throwaway
loop through it, so that the first hot loop's compile wouldn't pay the
code generator's cold start (about a millisecond). That pages in about
1.4 MB of Cranelift's code (and a little heap) in every process,
including the many that never compile anything: `-c pass` was 12.1 MB
peak RSS against 10.8 MB without it.

The engine is now built where it always was for a process that hadn't
warmed it, at the first compile. Peak RSS drops by about 1.1-1.3 MB for
`-c pass`, `import re`, `import asyncio` and `import typing, dataclasses,
enum`. Every suite workload compiles something, so its numbers don't
move (instructions geomean 1.001, RSS geomean 1.791x CPython against
1.793x).
---
 crates/weavepy-jit/src/engine.rs | 52 --------------------------------
 crates/weavepy-vm/src/tier2.rs   | 30 ++++--------------
 2 files changed, 6 insertions(+), 76 deletions(-)

diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs
index bea1ef30..634c4d7f 100644
--- a/crates/weavepy-jit/src/engine.rs
+++ b/crates/weavepy-jit/src/engine.rs
@@ -583,58 +583,6 @@ impl JitEngine {
         })
     }
 
-    /// Compile and discard a small counted loop, through the whole
-    /// pipeline a real compile takes (mid-end, register allocation,
-    /// emission, finalizing executable memory). A process's first compile
-    /// otherwise pays the code generator's cold start (its code paged in,
-    /// its tables built, the module's first memory mapped), about a
-    /// millisecond, on whatever hot loop first gets compiled.
-    pub fn warm_up(&mut self) {
-        use cranelift_codegen::ir::condcodes::IntCC;
-        use cranelift_codegen::ir::InstBuilder;
-        let int = types::I64;
-        self.ctx.func.signature.params.push(AbiParam::new(int));
-        self.ctx.func.signature.returns.push(AbiParam::new(int));
-        {
-            let mut b =
-                cranelift_frontend::FunctionBuilder::new(&mut self.ctx.func, &mut self.fbctx);
-            let entry = b.create_block();
-            let head = b.create_block();
-            let body = b.create_block();
-            let exit = b.create_block();
-            b.append_block_params_for_function_params(entry);
-            b.append_block_param(head, int);
-            b.append_block_param(head, int);
-            b.switch_to_block(entry);
-            let n = b.block_params(entry)[0];
-            let zero = b.ins().iconst(int, 0);
-            b.ins().jump(head, &[zero.into(), zero.into()]);
-            b.switch_to_block(head);
-            let (i, acc) = (b.block_params(head)[0], b.block_params(head)[1]);
-            let more = b.ins().icmp(IntCC::SignedLessThan, i, n);
-            b.ins().brif(more, body, &[], exit, &[]);
-            b.switch_to_block(body);
-            let acc2 = b.ins().iadd(acc, i);
-            let i2 = b.ins().iadd_imm(i, 1);
-            b.ins().jump(head, &[i2.into(), acc2.into()]);
-            b.switch_to_block(exit);
-            b.ins().return_(&[acc]);
-            b.seal_all_blocks();
-            b.finalize();
-        }
-        let name = format!("wpjit_warm_{}", self.next_id);
-        self.next_id += 1;
-        let defined = self
-            .module
-            .declare_function(&name, Linkage::Local, &self.ctx.func.signature)
-            .ok()
-            .and_then(|id| self.module.define_function(id, &mut self.ctx).ok());
-        self.module.clear_context(&mut self.ctx);
-        if defined.is_some() {
-            let _ = self.module.finalize_definitions();
-        }
-    }
-
     /// Analyze and compile a code object. `resolve` reports what each
     /// `LOAD_GLOBAL` name currently resolves to (see
     /// [`ResolvedGlobal`]); the caller must re-validate every resolution
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index 0dbafcbb..c2feeca4 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -1070,32 +1070,14 @@ fn import_compilation_budget() -> bool {
 }
 
 /// Mark interpreter start-up finished (see [`STARTUP_DONE`]).
+///
+/// The engine is still built at the first compile, not here: building it
+/// and running a compile through it pages in about 1.4 MB of the code
+/// generator, which a process that never compiles (a short script, most
+/// imports) would carry for nothing, to save the first compile about a
+/// millisecond.
 pub(crate) fn note_startup_finished() {
     STARTUP_DONE.store(true, std::sync::atomic::Ordering::Relaxed);
-    warm_engine();
-}
-
-/// Build this thread's engine and run one throwaway compile through it
-/// (see [`JitEngine::warm_up`]) as start-up ends, when the JIT is on: the
-/// first hot loop's compile then doesn't pay the code generator's cold
-/// start inside the work it speeds up.
-fn warm_engine() {
-    if jit_off_for_process() {
-        return;
-    }
-    JIT.with(|cell| {
-        let Ok(mut st) = cell.try_borrow_mut() else {
-            return;
-        };
-        if !st.enabled || st.engine.is_some() {
-            return;
-        }
-        st.engine = JitEngine::new();
-        match st.engine.as_mut() {
-            Some(engine) => engine.warm_up(),
-            None => st.enabled = false,
-        }
-    });
 }
 
 /// Whether a code object whose counter reached `counter` against

From fa9e3f81eb8b9a9decf892cd3b82ad97862a761c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 07:33:09 -0700
Subject: [PATCH 049/173] style: rustfmt the per-site table and shadow-scan
 changes

Formatting only: the accessor calls and signatures the per-site table
commit and the stdlib shadow scan introduced, as rustfmt lays them out.
---
 crates/weavepy-vm/src/frame_jit.rs | 16 ++-------
 crates/weavepy-vm/src/import.rs    |  7 ++--
 crates/weavepy-vm/src/lib.rs       | 52 +++++++++++++++++++-----------
 3 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 4ed0cb96..cbaa0d1f 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3531,13 +3531,7 @@ unsafe extern "C" fn h_stack_attr(
                         IC::LoadAttrInstance { key_idx, ver } if cls.attr_version.get() == ver => {
                             match inst.attr_peek_index(key_idx as usize) {
                                 Some((k, v)) if crate::slot_name_matches(code, arg, k) => {
-                                    crate::field_slot_note(
-                                        ext,
-                                        code,
-                                        pc,
-                                        inst,
-                                        key_idx,
-                                    );
+                                    crate::field_slot_note(ext, code, pc, inst, key_idx);
                                     Some(Interpreter::clone_operand(v))
                                 }
                                 _ => None,
@@ -4690,9 +4684,7 @@ unsafe extern "C" fn h_call(
         // fast half), without the shape checks below; a decline falls
         // through to them untouched.
         if let Object::Builtin(b) = &*base.add(start) {
-            let kind = ext
-                .method_slot(pc)
-                .and_then(|s| s.get_leaf(b));
+            let kind = ext.method_slot(pc).and_then(|s| s.get_leaf(b));
             if matches!(kind, Some(LeafKind::Isinstance | LeafKind::Fast(_)))
                 && Rc::strong_count(b) > 1
                 && !st.sw.is_null()
@@ -4964,9 +4956,7 @@ unsafe extern "C" fn h_call_leaf(
             let base = s.stack;
             let start = lenu - argc - 2;
             if let Object::Builtin(b) = &*base.add(start) {
-                let kind = x
-                    .method_slot(pcu)
-                    .and_then(|m| m.get_leaf(b));
+                let kind = x.method_slot(pcu).and_then(|m| m.get_leaf(b));
                 // A container's or string's method with a kernel, on the
                 // receiver in the self slot.
                 if let (Some(k), false) = (kind, matches!(&*base.add(start + 1), Object::Unbound)) {
diff --git a/crates/weavepy-vm/src/import.rs b/crates/weavepy-vm/src/import.rs
index d3160e9d..a13e172d 100644
--- a/crates/weavepy-vm/src/import.rs
+++ b/crates/weavepy-vm/src/import.rs
@@ -552,9 +552,10 @@ impl ModuleCache {
             return false;
         };
         let candidate = names.contains(std::ffi::OsStr::new(LANDMARK))
-            || names
-                .iter()
-                .any(|n| n.as_encoded_bytes().eq_ignore_ascii_case(LANDMARK.as_bytes()));
+            || names.iter().any(|n| {
+                n.as_encoded_bytes()
+                    .eq_ignore_ascii_case(LANDMARK.as_bytes())
+            });
         candidate && dir.join(LANDMARK).is_file()
     }
 
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ef720feb..ea33f17a 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -11299,9 +11299,7 @@ impl Interpreter {
         }
         macro_rules! mslots {
             ($cache:ident, $ext:expr) => {
-                *$cache.get_or_insert_with(|| {
-                    $ext.map_or(Sites::empty(), |e| e.method_sites())
-                })
+                *$cache.get_or_insert_with(|| $ext.map_or(Sites::empty(), |e| e.method_sites()))
             };
         }
         'reload: loop {
@@ -19782,13 +19780,7 @@ impl Interpreter {
             if !slot_name_matches_in(names, code, name_idx, key) {
                 return None;
             }
-            field_slot_note(
-                ext,
-                code,
-                cache_pc as usize,
-                inst,
-                key_idx,
-            );
+            field_slot_note(ext, code, cache_pc as usize, inst, key_idx);
             Some(value)
         } else {
             // SAFETY: the same rooted read as the dictionary path.
@@ -22070,10 +22062,7 @@ impl Interpreter {
         use weavepy_compiler::InlineCache as IC;
         let cls = inst.cls_raw();
         let ext = code_vm_ext(code);
-        if let Some((ver, idx)) = ext
-            .and_then(|e| e.field_slot(attr_pc))
-            .map(FieldSlot::get)
-        {
+        if let Some((ver, idx)) = ext.and_then(|e| e.field_slot(attr_pc)).map(FieldSlot::get) {
             if cls.attr_version.get() == ver && !crate::capi_watchers::dicts_active() {
                 // SAFETY: a store between two instructions (see `peek_mut`).
                 if let Some(slot) =
@@ -71978,7 +71967,13 @@ impl AttrPoly {
 #[inline]
 fn code_attr_poly(code: &CodeObject, cache_pc: u32) -> Option<&AttrPoly> {
     let ext = code_vm_ext(code)?;
-    warm_site(ext, code, &ext.attr_poly, cache_pc as usize, AttrPoly::empty)
+    warm_site(
+        ext,
+        code,
+        &ext.attr_poly,
+        cache_pc as usize,
+        AttrPoly::empty,
+    )
 }
 
 /// The state a `LOAD_GLOBAL` site last proved its cached slot against:
@@ -72229,7 +72224,8 @@ pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
     // SAFETY: GIL-serialized; the borrows end before any refill; a live
     // site's code is its live function's.
     let leaf = ext
-        .leaf_sites.get(pc)
+        .leaf_sites
+        .get(pc)
         .and_then(|s| unsafe { &*s.0.get() }.as_ref())
         .is_some_and(|site| {
             site.func.strong_count() > 0 && !(site.effect && code_calls(unsafe { &*site.code }))
@@ -72301,7 +72297,13 @@ unsafe fn field_slot_hit<'a>(
 /// (whose class passed the site's version check): kept only when that
 /// position is the split layout's, over the class's own names.
 #[inline(never)]
-fn field_slot_note(ext: &CodeConstObjects, code: &CodeObject, pc: usize, inst: &PyInstance, idx: u32) {
+fn field_slot_note(
+    ext: &CodeConstObjects,
+    code: &CodeObject,
+    pc: usize,
+    inst: &PyInstance,
+    idx: u32,
+) {
     // SAFETY: a read with nothing running (the caller's own guarded read).
     if unsafe { inst.split_field(idx as usize) }.is_none() {
         return;
@@ -72461,7 +72463,13 @@ fn drop_hot(v: Object) {
 #[inline]
 fn code_stamp_slot(code: &CodeObject, cache_pc: u32) -> Option<&StampSlot> {
     let ext = code_vm_ext(code)?;
-    warm_site(ext, code, &ext.stamp_slots, cache_pc as usize, StampSlot::empty)
+    warm_site(
+        ext,
+        code,
+        &ext.stamp_slots,
+        cache_pc as usize,
+        StampSlot::empty,
+    )
 }
 
 /// Tags of a `LOAD_ATTR` site's class-attribute stamp (`[attr_version,
@@ -73099,7 +73107,13 @@ impl MethodSlot {
 #[inline]
 fn code_method_slot(code: &CodeObject, cache_pc: u32) -> Option<&MethodSlot> {
     let ext = code_vm_ext(code)?;
-    warm_site(ext, code, &ext.method_slots, cache_pc as usize, MethodSlot::empty)
+    warm_site(
+        ext,
+        code,
+        &ext.method_slots,
+        cache_pc as usize,
+        MethodSlot::empty,
+    )
 }
 
 /// The plain function class `cls` (at attribute version `ver`) resolves

From d16e5fb139b860c36401500e729b547443a09fe6 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 04:45:23 -0700
Subject: [PATCH 050/173] perf: run a with block's exception exit as an inline
 call

WITH_EXCEPT_START called the exit method through `call`, which ran a
non-leaf `__exit__` as a nested frame with its own interpreter loop (and
compiled bodies exited back into it at every return). When the exit
method is a plain Python function of the manager, the core loop now
pushes the operands a `CALL 3` would see (the function, the manager,
the type, the exception and its traceback) and switches to the function
as an inline activation; its return lands past the WITH_EXCEPT_START and
a raise propagates as from a call there. A pure leaf `__exit__` keeps the
frameless evaluation `call` gives it.

contextlib.suppress swallowing an exception: 8177 -> 7215 instructions
per `with` (CPython 3164). Suite, per call: context_managers 0.982.
---
 crates/weavepy-vm/src/lib.rs            |  82 ++++++++
 tests/regrtest/test_with_exit_inline.py | 252 ++++++++++++++++++++++++
 2 files changed, 334 insertions(+)
 create mode 100644 tests/regrtest/test_with_exit_inline.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ea33f17a..6d25bd4b 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -15319,6 +15319,11 @@ impl Interpreter {
         if len < 5 {
             return false;
         }
+        if self.core_with_exit_inline(sw, pc) {
+            return true;
+        }
+        // SAFETY: as above (nothing was touched).
+        let frame = unsafe { &mut *sw.cur };
         // [exit_func, exit_self, lasti, prev, exc], as the full handler.
         let exc = frame.stack[len - 1].clone();
         let exit_self = frame.stack[len - 4].clone();
@@ -15355,6 +15360,83 @@ impl Interpreter {
         true
     }
 
+    /// [`Self::core_with_exit_lane`] for an exit method that is a plain
+    /// Python function of the manager (`LOAD_SPECIAL` left the function
+    /// under its instance): the call `exit(self, type, exc, tb)` runs as an
+    /// inline activation switched to here, as a `CALL 3` of the same
+    /// operands would, instead of nested through [`Self::call`]. Its
+    /// return lands on the stack past the `WITH_EXCEPT_START`; a raise
+    /// propagates from it as from a call there. `false` touches nothing.
+    #[inline(never)]
+    fn core_with_exit_inline(&mut self, sw: &mut CoreSwitch, pc: usize) -> bool {
+        if !self.inline_calls_ok() {
+            return false;
+        }
+        let code = {
+            // SAFETY: see `CoreSwitch`: the running activation is synced
+            // and unborrowed here.
+            let frame = unsafe { &*sw.cur };
+            let len = frame.stack.len();
+            // [exit_func, exit_self, lasti, prev, exc], as the full handler.
+            let (Object::Function(f), exit_self, Object::Instance(_)) = (
+                &frame.stack[len - 5],
+                &frame.stack[len - 4],
+                &frame.stack[len - 1],
+            ) else {
+                return false;
+            };
+            // A pure leaf is evaluated in place through `call` (see
+            // `call_pure_leaf`), cheaper than any activation.
+            if matches!(exit_self, Object::Unbound) || fn_is_pure_leaf(f) {
+                return false;
+            }
+            match Self::lean_call_shape_cells(f, 4, true) {
+                Some(shape) => shape,
+                None => return false,
+            }
+        };
+        let (code, missing) = code;
+        // Past the recursion limit the nested call raises.
+        let crate::recursion::Enter::Ok(guard) = crate::recursion::enter() else {
+            return false;
+        };
+        let mut tmp = None;
+        // SAFETY: see `CoreSwitch` (as in `core_call`): the running
+        // activation is the innermost; its handles are live and unborrowed.
+        let act = unsafe {
+            let depth = (*sw.inl).len();
+            let (frame, _, shell) = sw.activation(depth, &mut tmp);
+            let (frame, shell) = (&mut *frame, &mut *shell.cast::<QuietShell<'_>>());
+            let len = frame.stack.len();
+            let Object::Instance(inst) = &frame.stack[len - 1] else {
+                unreachable!("checked above")
+            };
+            let ty = Object::Type(inst.cls());
+            let tb = inst.slot_get("__traceback__").unwrap_or(Object::None);
+            let exc = frame.stack[len - 1].clone();
+            let exit_self = frame.stack[len - 4].clone();
+            let exit_func = frame.stack[len - 5].clone();
+            // Committed: the call's operands, as a `CALL 3` sees them.
+            frame.stack.extend([exit_func, exit_self, ty, exc, tb]);
+            let n = frame.stack.len();
+            let act = self.inline_slot();
+            // SAFETY: a parked slot's locals storage is its own, and empty.
+            let locals = &mut *act.frame.locals.as_ptr();
+            let callable =
+                Self::lean_call_fill(frame, &code, true, n - 4, n - 5, missing, locals);
+            frame.pc = pc as u32 + 1;
+            self.inline_bind(frame, shell, pc, act, code, callable, guard)
+        };
+        let mut act = act;
+        let callee: *mut Frame = &raw mut *act.frame;
+        // SAFETY: as above.
+        unsafe { push_fast(&mut *sw.inl, act) };
+        sw.cur = callee;
+        sw.scratch = usize::MAX;
+        sw.last = &raw mut sw.scratch;
+        true
+    }
+
     /// [`Self::core_builtin_lane`] for the `CALL_KW` at `pc` of a builtin
     /// function the lane admits (`sorted(xs, key=f)`), in `sw`'s synced
     /// running activation: the operands leave the stack, the call runs
diff --git a/tests/regrtest/test_with_exit_inline.py b/tests/regrtest/test_with_exit_inline.py
new file mode 100644
index 00000000..09154392
--- /dev/null
+++ b/tests/regrtest/test_with_exit_inline.py
@@ -0,0 +1,252 @@
+"""A `with` block's exit on an exception: when the manager's `__exit__` is a
+plain Python function (not a frameless leaf), `WITH_EXCEPT_START` runs it as
+an inline activation, as a `CALL` of the same operands would (see
+`core_with_exit_inline` in the VM), instead of nesting the call.
+
+Each case runs warm (loops long enough to compile the callers) and checks
+what the nested call did: the arguments, the handled exception, the
+caller's frame, suppression and re-raising, raises out of `__exit__` with
+their context and traceback, and finalizer timing.
+"""
+
+import sys
+import traceback
+import unittest
+
+WARM = 3000
+
+
+class Recorder:
+    """An `__exit__` too large to be a frameless leaf."""
+
+    def __init__(self, suppress):
+        self.suppress = suppress
+        self.seen = []
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exctype, excinst, exctb):
+        if exctype is None:
+            return None
+        self.seen.append((exctype, excinst, exctb is excinst.__traceback__))
+        if issubclass(exctype, BaseExceptionGroup):
+            match, rest = excinst.split(KeyError)
+            if rest is None:
+                return True
+            raise rest
+        return self.suppress
+
+
+class Varargs:
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.args = args
+        try:
+            pass
+        finally:
+            self.done = True
+        return True
+
+
+class Inspector:
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exctype, excinst, exctb):
+        caller = sys._getframe(1)
+        self.line = caller.f_lineno
+        self.name = caller.f_code.co_name
+        self.info = sys.exc_info()[1]
+        for _ in ():
+            raise
+        return True
+
+
+class Raiser:
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exctype, excinst, exctb):
+        if exctype is not None:
+            raise ValueError("from exit") from None
+        return False
+
+
+class Chained:
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exctype, excinst, exctb):
+        if exctype is not None:
+            raise ValueError("from exit")
+        return False
+
+
+class Mortal:
+    log = []
+
+    def __del__(self):
+        Mortal.log.append("del")
+
+
+class Holder:
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exctype, excinst, exctb):
+        self.held = excinst
+        for _ in ():
+            raise
+        return True
+
+
+class WithExitInlineTests(unittest.TestCase):
+    def test_suppress_warm(self):
+        r = Recorder(True)
+        d = {}
+        for i in range(WARM):
+            with r:
+                d[i]
+        self.assertEqual(len(r.seen), WARM)
+        t, e, same_tb = r.seen[-1]
+        self.assertIs(t, KeyError)
+        self.assertIsInstance(e, KeyError)
+        self.assertEqual(e.args, (WARM - 1,))
+        self.assertTrue(same_tb)
+
+    def test_reraise_warm(self):
+        r = Recorder(False)
+        caught = 0
+        for i in range(WARM):
+            try:
+                with r:
+                    raise IndexError(i)
+            except IndexError as e:
+                self.assertEqual(e.args, (i,))
+                caught += 1
+        self.assertEqual(caught, WARM)
+
+    def test_reraise_keeps_traceback(self):
+        r = Recorder(False)
+
+        def body():
+            with r:
+                raise IndexError("deep")
+
+        for _ in range(WARM):
+            try:
+                body()
+            except IndexError as e:
+                tb = e.__traceback__
+        names = [f.name for f in traceback.extract_tb(tb)]
+        self.assertEqual(names[-1], "body")
+        self.assertIn("raise IndexError", traceback.format_tb(tb)[-1])
+
+    def test_group_split(self):
+        r = Recorder(False)
+        for _ in range(WARM):
+            with r:
+                raise ExceptionGroup("g", [KeyError(1)])
+        with self.assertRaises(ExceptionGroup) as cm:
+            with r:
+                raise ExceptionGroup("g", [KeyError(1), ValueError(2)])
+        self.assertEqual(len(cm.exception.exceptions), 1)
+        self.assertIsInstance(cm.exception.exceptions[0], ValueError)
+
+    def test_varargs_exit(self):
+        v = Varargs()
+        for i in range(WARM):
+            with v:
+                raise KeyError(i)
+        self.assertEqual(len(v.args), 3)
+        self.assertIs(v.args[0], KeyError)
+        self.assertEqual(v.args[1].args, (WARM - 1,))
+        self.assertIs(v.args[2], v.args[1].__traceback__)
+        self.assertTrue(v.done)
+
+    def test_caller_frame_and_exc_info(self):
+        insp = Inspector()
+        for i in range(WARM):
+            with insp:
+                expected = sys._getframe().f_lineno - 1
+                raise KeyError(i)
+        self.assertEqual(insp.line, expected)
+        self.assertEqual(insp.name, "test_caller_frame_and_exc_info")
+        self.assertIsInstance(insp.info, KeyError)
+        self.assertEqual(insp.info.args, (WARM - 1,))
+        self.assertIsNone(sys.exc_info()[1])
+
+    def test_raise_from_exit_chains_context(self):
+        c = Chained()
+        for i in range(WARM):
+            try:
+                with c:
+                    raise KeyError(i)
+            except ValueError as e:
+                err = e
+        self.assertEqual(err.args, ("from exit",))
+        self.assertIsInstance(err.__context__, KeyError)
+        self.assertEqual(err.__context__.args, (WARM - 1,))
+        names = [f.name for f in traceback.extract_tb(err.__traceback__)]
+        self.assertEqual(names[-1], "__exit__")
+        self.assertIn("test_raise_from_exit_chains_context", names)
+
+    def test_raise_from_exit_inside_handler(self):
+        c = Raiser()
+        hits = 0
+        for i in range(WARM):
+            try:
+                with c:
+                    raise KeyError(i)
+            except ValueError:
+                hits += 1
+        self.assertEqual(hits, WARM)
+        self.assertIsNone(sys.exc_info()[1])
+
+    def test_exception_released_promptly(self):
+        h = Holder()
+        for _ in range(WARM):
+            with h:
+                raise KeyError(Mortal())
+        Mortal.log.clear()
+        h.held = None
+        self.assertEqual(Mortal.log, ["del"])
+
+    def test_nested_withs(self):
+        outer = Recorder(True)
+        inner = Recorder(False)
+        for i in range(WARM):
+            with outer:
+                with inner:
+                    raise LookupError(i)
+        self.assertEqual(len(inner.seen), WARM)
+        self.assertEqual(len(outer.seen), WARM)
+        self.assertIs(inner.seen[-1][1], outer.seen[-1][1])
+
+    def test_recursion_limit(self):
+        r = Recorder(True)
+
+        def deep(n):
+            with r:
+                if n:
+                    deep(n - 1)
+                raise KeyError(n)
+
+        old = sys.getrecursionlimit()
+        try:
+            sys.setrecursionlimit(200)
+            for _ in range(50):
+                try:
+                    deep(1000)
+                except RecursionError:
+                    pass
+        finally:
+            sys.setrecursionlimit(old)
+        deep(5)
+
+
+if __name__ == "__main__":
+    unittest.main()

From b889414bddab6930ffc1a74ae2675f45a1865afa Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 05:53:33 -0700
Subject: [PATCH 051/173] perf: cheaper generator creation, member loads and
 native resumes

A generator snapshots its function's `__name__` and `__qualname__`. Once
something read the function's getset slots (as `functools.wraps` does,
which every `@contextmanager` function gets), each generator it made
probed the slots for both names and compared them with the code's. The
function now remembers the slots' mutation stamp at which both were the
code's own interned names, so later generators settle with one compare;
rebinding `__code__` clears it.

`cls.__doc__` under a Python metaclass (`ABCMeta`, as in
`_GeneratorContextManagerBase.__init__`'s `type(self).__doc__`) hashed
the name for each of its two dict probes; it probes with the site's
pre-hashed name.

A native consumer (`sum`, `max`, a `for` in native code) resumes a
generator through a quiet run per item, and each run grew a fresh
activation stack and freed it. Finished runs leave their emptied stack
for the next.

A compiled `__slots__` member load's cache hit peeks at the slots
instead of taking a guard, and reads the few-members layout in line.

Instructions retired vs the previous commit (suite, per call):
context_managers 0.989, generators_tree 0.979, async_tree 0.997,
coroutines 1.007 (noise; probes unchanged).
---
 crates/weavepy-vm/src/builtins.rs             |  2 +
 crates/weavepy-vm/src/frame_jit.rs            |  6 +-
 crates/weavepy-vm/src/lib.rs                  | 25 ++++-
 crates/weavepy-vm/src/object.rs               | 34 +++++++
 crates/weavepy-vm/src/types.rs                | 15 +++
 .../regrtest/test_generator_name_snapshot.py  | 99 +++++++++++++++++++
 6 files changed, 175 insertions(+), 6 deletions(-)
 create mode 100644 tests/regrtest/test_generator_name_snapshot.py

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 98003a7e..327896a1 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -5339,6 +5339,7 @@ fn attr_set(obj: &Object, name: &str, value: Object) -> Result<(), RuntimeError>
                     )));
                 }
                 *f.code.borrow_mut() = c;
+                f.names_kept.set(0);
             } else if crate::object::is_function_slot(name) {
                 f.set_slot(name, value);
             } else {
@@ -10640,6 +10641,7 @@ fn b_mark_iterable_coroutine(args: &[Object]) -> Result<Object, RuntimeError> {
         closure_cells: std::sync::OnceLock::new(),
         // The copied slot store carries any override along.
         defaults_override: crate::object::OverrideFlag::new(f.defaults_maybe_overridden()),
+        names_kept: crate::sync::Cell::new(0),
         gc_slot: crate::gc_trace::GcSlot::new(),
     };
     Ok(Object::Function(Rc::new(marked)))
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index cbaa0d1f..2bd63b37 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3469,8 +3469,10 @@ unsafe fn slot_hit(
     if ver == 0 || inst.cls_raw().attr_version.get() != ver {
         return None;
     }
-    let slots = inst.slots.try_borrow().ok()?;
-    let (k, v) = slots.get_index(cache.idx.get() as usize)?;
+    // SAFETY: a read between two instructions, running no code (see
+    // `GilCell::peek`); the view ends with the value's clone.
+    let slots = unsafe { inst.slots.peek() }?;
+    let (k, v) = slots.get_index_hot(cache.idx.get() as usize)?;
     crate::slot_name_matches(code, code.instructions[pc].arg, k)
         .then(|| Interpreter::clone_operand(v))
 }
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 6d25bd4b..fe54e8e6 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -930,6 +930,11 @@ pub struct Interpreter {
     /// invalidate on every growth.
     #[allow(clippy::vec_box)]
     inline_pool: Vec<Box<InlineAct>>,
+    /// Emptied activation stacks of finished quiet runs (see
+    /// [`Self::quiet_run`]), for the next run: a generator its native
+    /// consumer resumes item by item starts a quiet run per item.
+    #[allow(clippy::vec_box)]
+    inl_spares: Vec<Vec<Box<InlineAct>>>,
     /// Finished generators' frame boxes, their frames dropped, for the
     /// next generators' frames (see [`Self::box_gen_frame`]).
     #[allow(clippy::vec_box)]
@@ -1240,6 +1245,7 @@ impl Default for Interpreter {
             internal_import_depth: 0,
             frame_locals_pool: ThreadCell::new(Vec::new()),
             inline_pool: Vec::new(),
+            inl_spares: Vec::new(),
             gen_box_pool: ThreadCell::new(Vec::new()),
             tuple_pool: ThreadCell::new(std::array::from_fn(|_| Vec::new())),
             frame_shell_pool: ThreadCell::new(Vec::new()),
@@ -1370,6 +1376,7 @@ impl Interpreter {
             internal_import_depth: 0,
             frame_locals_pool: ThreadCell::new(Vec::new()),
             inline_pool: Vec::new(),
+            inl_spares: Vec::new(),
             gen_box_pool: ThreadCell::new(Vec::new()),
             tuple_pool: ThreadCell::new(std::array::from_fn(|_| Vec::new())),
             frame_shell_pool: ThreadCell::new(Vec::new()),
@@ -7462,8 +7469,9 @@ impl Interpreter {
         let maybe_dead = gc_trace::maybe_dead_flag();
         // The root activation's last executed pc (its `prev_pc` feed).
         let mut last = usize::MAX;
-        // The inline activations, innermost last.
-        let mut inl: Vec<Box<InlineAct>> = Vec::new();
+        // The inline activations, innermost last (in a finished run's
+        // emptied vector, when one is spare).
+        let mut inl: Vec<Box<InlineAct>> = self.inl_spares.pop().unwrap_or_default();
         // Every access below goes through these handles (see
         // `CoreSwitch`): the core loop's in-loop calls and returns move
         // the running activation through the same ones.
@@ -7535,6 +7543,11 @@ impl Interpreter {
                 FrameEv::Switched(stop) => QuietEntry::Stop(stop),
                 FrameEv::Exit(exit) => {
                     let Some(mut done) = inl.pop() else {
+                        // (Empty: the vector goes back for the next run.)
+                        const SPARES_CAP: usize = 16;
+                        if inl.capacity() != 0 && self.inl_spares.len() < SPARES_CAP {
+                            self.inl_spares.push(std::mem::take(inl));
+                        }
                         return exit;
                     };
                     if done.gen.is_some() {
@@ -27046,7 +27059,8 @@ impl Interpreter {
                 .is_some_and(|n| n == "__doc__")
         {
             let plain = |v: Option<&Object>| matches!(v, Some(Object::Str(_) | Object::None));
-            let key = crate::object::StrKey("__doc__");
+            // (The site's name, hashed once.)
+            let key = code_name_key(code, name_idx)?;
             if !Self::plain_metaclass(cls) {
                 // SAFETY: GIL-serialized raw read, as in `plain_metaclass`.
                 let meta = unsafe { (*cls.metaclass.as_ptr()).as_ref() }?;
@@ -48011,6 +48025,7 @@ impl Interpreter {
                         // was made from.
                         f.slots();
                         *f.code.borrow_mut() = c;
+                        f.names_kept.set(0);
                         crate::rare_events::bump(crate::rare_events::FUNC_MODIFICATION);
                         if crate::capi_watchers::funcs_active() {
                             crate::capi_watchers::func_event("MODIFY_CODE", obj, &new_code);
@@ -52265,6 +52280,7 @@ impl Interpreter {
             slot_seed: RefCell::new(None),
             closure_cells: std::sync::OnceLock::new(),
             defaults_override: crate::object::OverrideFlag::new(false),
+            names_kept: crate::sync::Cell::new(0),
             gc_slot: crate::gc_trace::GcSlot::new(),
         })))
     }
@@ -56468,7 +56484,7 @@ impl Interpreter {
         // the generator is made.
         let gen = Rc::new_cyclic(|owner| {
             frame.gen_owner = Some(owner.clone());
-            if f.names_seeded() {
+            if f.code_names_kept(|| code_gen_names(code)) {
                 // The code's names, which the generator reads by default.
                 PyGenerator::new(kind, gen_code, frame)
             } else {
@@ -74140,6 +74156,7 @@ fn new_function(
         gc_slot: crate::gc_trace::GcSlot::new(),
         closure_cells: std::sync::OnceLock::new(),
         defaults_override: crate::object::OverrideFlag::new(false),
+        names_kept: crate::sync::Cell::new(0),
     };
     if let Some(ann) = annotations {
         f.slots()
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 0b0f38b1..58600920 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -5033,6 +5033,11 @@ pub struct PyFunction {
     /// slot-store probe for the overwhelmingly common function that never
     /// had them overridden.
     pub defaults_override: OverrideFlag,
+    /// The [`Self::slots`] state (its mutation stamp, `0` for none) in
+    /// which `__name__` and `__qualname__` were last found to be the
+    /// current code's own interned names (see [`Self::code_names_kept`]).
+    /// Cleared when `__code__` is rebound.
+    pub names_kept: crate::sync::Cell<u64>,
     /// The function's slot in the cycle collector's registry (see
     /// [`crate::gc_trace::GcSlot`]).
     pub gc_slot: crate::gc_trace::GcSlot,
@@ -5244,6 +5249,35 @@ impl PyFunction {
         )
     }
 
+    /// Whether `__name__` and `__qualname__` are the very objects `names`
+    /// gives (the current code's interned names, which a generator made
+    /// by this function then reports by default): the slots' seed, or
+    /// slots in a state already found to hold them, answer without a
+    /// probe.
+    pub fn code_names_kept(&self, names: impl FnOnce() -> (Object, Object)) -> bool {
+        if self.names_seeded() {
+            return true;
+        }
+        let Some(slots) = self.slots_raw.get() else {
+            return false;
+        };
+        let stamp = match slots.try_borrow() {
+            Ok(s) => s.mutation_stamp(),
+            Err(_) => return false,
+        };
+        if stamp == self.names_kept.get() {
+            return true;
+        }
+        let (name, qualname) = self.name_objects();
+        let (code_name, code_qualname) = names();
+        let kept = name.is_some_and(|n| n.is_same(&code_name))
+            && qualname.is_some_and(|q| q.is_same(&code_qualname));
+        if kept {
+            self.names_kept.set(stamp);
+        }
+        kept
+    }
+
     /// Whether the function's slots are still its definition-time seed
     /// (so `__name__` and `__qualname__` are its code's).
     pub fn names_seeded(&self) -> bool {
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 8f48a7cf..4d0f1016 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -2527,6 +2527,16 @@ impl SlotStorage {
         }
     }
 
+    /// [`Self::get_index`] with the common layout of a few `__slots__`
+    /// members read in line (a cached member load's hot path).
+    #[inline(always)]
+    pub fn get_index_hot(&self, index: usize) -> Option<(&DictKey, &Object)> {
+        if let SlotData::Small(entries) = &self.data {
+            return entries.get(index).map(|(key, value)| (key, value));
+        }
+        self.get_index(index)
+    }
+
     /// Read an ordered entry. Callers caching an index must also validate
     /// its key; the class identity alone does not determine slot order.
     #[inline]
@@ -3023,6 +3033,11 @@ impl SlotStorage {
         self.0.get_index(index)
     }
 
+    #[inline]
+    pub fn get_index_hot(&self, index: usize) -> Option<(&DictKey, &Object)> {
+        self.0.get_index(index)
+    }
+
     #[inline]
     pub fn get_index_mut(&mut self, index: usize) -> Option<(&DictKey, &mut Object)> {
         self.0.get_index_mut(index)
diff --git a/tests/regrtest/test_generator_name_snapshot.py b/tests/regrtest/test_generator_name_snapshot.py
new file mode 100644
index 00000000..6747dffc
--- /dev/null
+++ b/tests/regrtest/test_generator_name_snapshot.py
@@ -0,0 +1,99 @@
+"""A generator snapshots its function's `__name__` and `__qualname__` at
+creation. A function whose getset slots were read (as `functools.wraps`
+reads them) remembers the slots' state in which it last found its code's
+own names there, so later generators skip the comparison (see
+`PyFunction::code_names_kept` in the VM). Each case makes generators warm,
+then renames or swaps the code, and checks what new generators report.
+"""
+
+import functools
+import unittest
+
+WARM = 2000
+
+
+def plain():
+    yield 1
+
+
+def other():
+    yield 2
+
+
+class GeneratorNameSnapshotTests(unittest.TestCase):
+    def make(self):
+        def gen(x):
+            yield x
+
+        # Reading the slots the way `wraps` does materializes them.
+        @functools.wraps(gen)
+        def wrapper(*a):
+            return gen(*a)
+
+        return gen, wrapper
+
+    def test_wrapped_names_warm(self):
+        gen, wrapper = self.make()
+        for i in range(WARM):
+            g = wrapper(i)
+        self.assertEqual(g.__name__, "gen")
+        self.assertTrue(g.__qualname__.endswith("make.<locals>.gen"))
+        self.assertIs(g.__name__, gen.__name__)
+        self.assertIs(g.__qualname__, gen.__qualname__)
+
+    def test_rename_after_warm(self):
+        gen, wrapper = self.make()
+        for i in range(WARM):
+            wrapper(i)
+        gen.__name__ = "renamed"
+        g = wrapper(0)
+        self.assertEqual(g.__name__, "renamed")
+        self.assertIs(g.__name__, gen.__name__)
+        gen.__qualname__ = "Q.renamed"
+        g = wrapper(0)
+        self.assertEqual(g.__qualname__, "Q.renamed")
+        for i in range(WARM):
+            g = wrapper(i)
+        self.assertEqual((g.__name__, g.__qualname__), ("renamed", "Q.renamed"))
+
+    def test_rename_back_to_equal_text(self):
+        gen, wrapper = self.make()
+        for i in range(WARM):
+            wrapper(i)
+        text = "".join(["g", "en"])
+        gen.__name__ = text
+        g = wrapper(0)
+        self.assertEqual(g.__name__, "gen")
+        self.assertIs(g.__name__, gen.__name__)
+
+    def test_code_swap(self):
+        def f():
+            yield 1
+
+        functools.update_wrapper(lambda: None, f)
+        for _ in range(WARM):
+            g = f()
+        self.assertEqual(g.__name__, "f")
+        f.__code__ = other.__code__
+        g = f()
+        self.assertEqual(next(g), 2)
+        self.assertEqual(g.__name__, f.__name__)
+        self.assertEqual(g.__qualname__, f.__qualname__)
+
+    def test_unread_slots(self):
+        for _ in range(WARM):
+            g = plain()
+        self.assertEqual(g.__name__, "plain")
+        self.assertEqual(g.__qualname__, "plain")
+
+    def test_generator_name_reassigned(self):
+        gen, wrapper = self.make()
+        for i in range(WARM):
+            g = wrapper(i)
+        g.__name__ = "mine"
+        self.assertEqual(g.__name__, "mine")
+        self.assertEqual(wrapper(1).__name__, "gen")
+
+
+if __name__ == "__main__":
+    unittest.main()

From 5524d4b956b76e9e2531bd1038d521b07f79e5a4 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 06:31:37 -0700
Subject: [PATCH 052/173] perf: share the split-dict tables that attribute
 deletions make

Deleting any but the last attribute of an instance with a split
`__dict__` copied the remaining names into a private table (a 30-slot
`SharedKeys`, built, moved into an `Rc` and freed with the instance).
A table now remembers the tables its deletions made, keyed by how many
names the instance held and which one went, so instances that delete
the same attributes in the same order (`_GeneratorContextManager.
__enter__`'s `del self.args, self.kwds, self.func`) share them as they
share their class's names. A shared table only grows by appends, which
leave the names it was made from in place; a table still private to one
instance closes up in place as before, forgetting what it derived.

A `DELETE_ATTR` site also remembers, per class version, that the class
leaves its name to the instance's own dictionary under the default
`__delattr__`, so the class-attribute and `__delattr__` lookups run once.

Three deletions from a fresh five-attribute instance: 7028 -> 5993
instructions per iteration (CPython 2569). Suite, per call:
context_managers 0.981, async_tree 0.994.
---
 crates/weavepy-vm/src/inst_dict.rs            | 53 ++++++++--
 crates/weavepy-vm/src/lib.rs                  | 43 ++++++---
 .../test_split_dict_shared_deletions.py       | 96 +++++++++++++++++++
 3 files changed, 170 insertions(+), 22 deletions(-)
 create mode 100644 tests/regrtest/test_split_dict_shared_deletions.py

diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index 2515c78b..41ab8786 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -41,6 +41,10 @@ pub struct SharedKeys {
     keys: [UnsafeCell<MaybeUninit<DictKey>>; SHARED_KEYS_CAP],
     /// Each published name's Python hash.
     hashes: [UnsafeCell<i64>; SHARED_KEYS_CAP],
+    /// Tables made from this one by a deletion (see
+    /// [`Self::without`]): `(n, i, table)` for the first `n` names less
+    /// the `i`th. Touched only under the GIL.
+    derived: UnsafeCell<Vec<(u32, u32, Rc<SharedKeys>)>>,
 }
 
 // SAFETY: names are published with release/acquire ordering and are
@@ -64,6 +68,7 @@ impl Default for SharedKeys {
             filter: AtomicU64::new(0),
             keys: [const { UnsafeCell::new(MaybeUninit::uninit()) }; SHARED_KEYS_CAP],
             hashes: [const { UnsafeCell::new(0) }; SHARED_KEYS_CAP],
+            derived: UnsafeCell::new(Vec::new()),
         }
     }
 }
@@ -149,6 +154,39 @@ impl SharedKeys {
             filter |= 1 << (*h.get_mut() & 63);
         }
         *self.filter.get_mut() = filter;
+        // The tables derived from the old names no longer derive from these.
+        self.derived.get_mut().clear();
+    }
+
+    /// The first `n` names less the `i`th (`i + 1 < n <= len`), as a table
+    /// of their own: the one an earlier deletion of the same shape made,
+    /// when this table remembers it, so instances that delete the same
+    /// attributes in the same order share their tables, as they share
+    /// their class's (deletions in `__enter__`, say). A remembered table
+    /// may have grown since (appends of later stores), but its first
+    /// `n - 1` names stay these. The caller holds the GIL.
+    fn without(&self, n: usize, i: usize) -> Rc<SharedKeys> {
+        const DERIVED_CAP: usize = 8;
+        // SAFETY: GIL-serialized; nothing below reaches this list again
+        // (building a table runs no code).
+        let derived = unsafe { &mut *self.derived.get() };
+        if let Some((_, _, t)) = derived
+            .iter()
+            .find(|&&(dn, di, _)| dn as usize == n && di as usize == i)
+        {
+            return t.clone();
+        }
+        let fresh = SharedKeys::default();
+        for k in (0..n).filter(|&k| k != i) {
+            let key = self.get(k).expect("a set value's name").clone();
+            // SAFETY: slot `k` is published (below the length).
+            fresh.push_key(key, unsafe { *self.hashes[k].get() });
+        }
+        let fresh = Rc::new(fresh);
+        if derived.len() < DERIVED_CAP {
+            derived.push((n as u32, i as u32, fresh.clone()));
+        }
+        fresh
     }
 
     /// [`Self::push`] of a name key whose Python hash is `hash`.
@@ -811,7 +849,10 @@ impl SplitValues {
     /// Remove attribute `name`, returning its value (`None` when it is not
     /// set). The last value simply leaves (the instance then holds one
     /// name fewer of the same table); removing any other lays the rest out
-    /// over a private copy of their names, in order. Compiled code checks
+    /// over a table of their names, in order (the one other instances that
+    /// deleted the same way share, see [`SharedKeys::without`], or one
+    /// private to this instance, which closes up in place on the next
+    /// deletion). Compiled code checks
     /// an instance's names against its class's before using a position, so
     /// such an instance takes the general paths from then on, as one with
     /// a real dictionary does, without materializing one (CPython's split
@@ -832,14 +873,8 @@ impl SplitValues {
                     // Names already private to this instance (an earlier
                     // deletion's) close up in place.
                     Some(own) => own.remove_at(i),
-                    None => {
-                        let fresh = SharedKeys::default();
-                        for k in (0..n).filter(|&k| k != i) {
-                            let key = keys.get(k).expect("a set value's name").clone();
-                            fresh.push_key(key, *keys.hashes[k].get());
-                        }
-                        keys = Rc::new(fresh);
-                    }
+                    // Shared names: the table for the rest of them.
+                    None => keys = keys.without(n, i),
                 }
                 (*h).keys = Rc::into_raw(keys);
                 std::ptr::copy(values.add(i + 1), values.add(i), n - i - 1);
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index fe54e8e6..f329fc3b 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -14767,7 +14767,7 @@ impl Interpreter {
                     OpCode::DeleteAttr => {
                         // SAFETY: the `len` slots at `base` are the core
                         // loop's initialized stack.
-                        match unsafe { self.core_delete_attr(code, ins, base, len) } {
+                        match unsafe { self.core_delete_attr(code, ins, pc, base, len) } {
                             Some(n) => len = n,
                             None => break None,
                         }
@@ -17555,6 +17555,7 @@ impl Interpreter {
         &mut self,
         code: &CodeObject,
         ins: weavepy_compiler::Instruction,
+        pc: usize,
         base: *mut Object,
         len: usize,
     ) -> Option<usize> {
@@ -17570,20 +17571,30 @@ impl Interpreter {
         };
         let cls = inst.cls_raw();
         // (The full arm's conditions; see there.)
-        if inst.native.get().is_some()
-            || key.starts_with("__")
-            || &**key == "_CHUNK_SIZE"
-            || cls.flags.is_exception
-            || cls.forbids_dict
-            || crate::capi_watchers::dicts_active()
-            || !Self::default_delattr(cls)
-            || !matches!(
-                Self::leaf_class_attr(code, cls, ins.arg),
-                Some(LeafAttr::InstanceOnly)
-            )
-        {
+        if inst.native.get().is_some() || crate::capi_watchers::dicts_active() {
             return None;
         }
+        // The rest depend on the name and the class alone, settled once
+        // per class version at the site (versions are process-unique).
+        let ver = cls.attr_version.get();
+        let stamp = code_stamp_slot(code, pc as u32);
+        if !stamp.is_some_and(|s| s.get() == [ver, DELETE_OWN_ATTR, 0]) {
+            if key.starts_with("__")
+                || &**key == "_CHUNK_SIZE"
+                || cls.flags.is_exception
+                || cls.forbids_dict
+                || !Self::default_delattr(cls)
+                || !matches!(
+                    Self::leaf_class_attr(code, cls, ins.arg),
+                    Some(LeafAttr::InstanceOnly)
+                )
+            {
+                return None;
+            }
+            if let Some(s) = stamp {
+                s.set([ver, DELETE_OWN_ATTR, 0]);
+            }
+        }
         let removed = match inst.split_remove(key) {
             Some(removed) => removed?,
             None => inst
@@ -72600,6 +72611,12 @@ const TUPLE_FIELD: u64 = 0x5ca1_a770_0000_0005;
 /// data descriptor, so no instance dictionary shadows it.
 const LAID_OUT_SLOT: u64 = 0x5ca1_a770_0000_0006;
 
+/// A `DELETE_ATTR` site's stamp tag: the class at that version deletes the
+/// site's name from an instance's own dictionary by the default
+/// `__delattr__`, with no class attribute of that name in the way (see
+/// [`Interpreter::core_delete_attr`]).
+const DELETE_OWN_ATTR: u64 = 0x5ca1_a770_0000_0007;
+
 /// The named tuple field a `LOAD_ATTR` site's stamp remembers for
 /// `inst`'s class at its current version (see [`TUPLE_FIELD`]): the item
 /// of `inst`'s tuple, or `None` (an index out of range is the
diff --git a/tests/regrtest/test_split_dict_shared_deletions.py b/tests/regrtest/test_split_dict_shared_deletions.py
new file mode 100644
index 00000000..728c8f74
--- /dev/null
+++ b/tests/regrtest/test_split_dict_shared_deletions.py
@@ -0,0 +1,96 @@
+"""Deleting an attribute from an instance with a split `__dict__` lays the
+remaining values out over a table of their names. Instances that delete
+the same attributes in the same order share those tables (see
+`SharedKeys::without` in the VM), so each case checks that one instance's
+later stores and deletions never show through another's `__dict__`,
+attribute reads, or iteration order, in loops long enough to compile.
+"""
+
+import unittest
+
+WARM = 2000
+
+
+class Box:
+    def __init__(self, a, b, c, d):
+        self.a = a
+        self.b = b
+        self.c = c
+        self.d = d
+
+
+class SharedDeletionTests(unittest.TestCase):
+    def test_same_deletions_warm(self):
+        for i in range(WARM):
+            o = Box(i, 2, 3, 4)
+            del o.b, o.c
+            self.assertEqual(o.__dict__, {"a": i, "d": 4})
+            self.assertEqual(o.a + o.d, i + 4)
+            self.assertFalse(hasattr(o, "b"))
+            self.assertFalse(hasattr(o, "c"))
+
+    def test_store_after_shared_deletion(self):
+        first = Box(1, 2, 3, 4)
+        del first.b
+        first.z = 26
+        second = Box(5, 6, 7, 8)
+        del second.b
+        self.assertEqual(list(second.__dict__), ["a", "c", "d"])
+        self.assertFalse(hasattr(second, "z"))
+        second.w = 23
+        self.assertEqual(list(second.__dict__), ["a", "c", "d", "w"])
+        self.assertEqual(list(first.__dict__), ["a", "c", "d", "z"])
+        self.assertEqual(second.w, 23)
+        self.assertFalse(hasattr(first, "w"))
+        third = Box(9, 10, 11, 12)
+        del third.b
+        third.z = 0
+        self.assertEqual(third.__dict__, {"a": 9, "c": 11, "d": 12, "z": 0})
+        self.assertEqual(first.z, 26)
+
+    def test_different_orders(self):
+        for i in range(WARM):
+            o = Box(1, 2, 3, 4)
+            p = Box(5, 6, 7, 8)
+            del o.a, o.c
+            del p.c, p.a
+            self.assertEqual(o.__dict__, {"b": 2, "d": 4})
+            self.assertEqual(p.__dict__, {"b": 6, "d": 8})
+            del o.d
+            self.assertEqual(o.__dict__, {"b": 2})
+            self.assertEqual(p.__dict__, {"b": 6, "d": 8})
+            p.a = i
+            self.assertEqual(list(p.__dict__), ["b", "d", "a"])
+            self.assertFalse(hasattr(o, "a"))
+
+    def test_delete_then_readd(self):
+        for i in range(WARM):
+            o = Box(1, 2, 3, 4)
+            del o.a
+            o.a = i
+            self.assertEqual(list(o.__dict__), ["b", "c", "d", "a"])
+            self.assertEqual(o.a, i)
+            del o.c
+            self.assertEqual(list(o.__dict__), ["b", "d", "a"])
+
+    def test_many_patterns(self):
+        names = "abcd"
+        for i in range(WARM):
+            o = Box(1, 2, 3, 4)
+            k = names[i % 3]
+            delattr(o, k)
+            expected = {n: v for n, v in zip(names, (1, 2, 3, 4)) if n != k}
+            self.assertEqual(o.__dict__, expected)
+            vars(o)["e"] = 5
+            self.assertEqual(o.e, 5)
+
+    def test_missing_attribute(self):
+        o = Box(1, 2, 3, 4)
+        del o.b
+        with self.assertRaises(AttributeError):
+            del o.b
+        self.assertEqual(o.__dict__, {"a": 1, "c": 3, "d": 4})
+
+
+if __name__ == "__main__":
+    unittest.main()

From 89d4d363d75ad3bb48d178cc22fec614d4edc526 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 07:11:52 -0700
Subject: [PATCH 053/173] perf: fold sum over generator iterables and resume
 next(gen, default) inline

`sum(obj)` drains a generator through the lean resume that folds each
yield into the running total in place, but only when `obj` was the
generator itself. An iterable whose `__iter__` is a generator function
(a tree walked with `yield from`) went item by item through `iter_next`,
a quiet run per item. `sum` now takes `iter(obj)` first and drains any
generator it gets the same way.

`next(gen, default)` went through the builtin lane and a nested quiet
run. Like `next(gen)`, it now resumes the generator as an inline
activation (`InlineResume::NextDefault`), with the default waiting atop
the caller's stack: a yield replaces it and an exhausted generator
leaves it as the call's result. The core loop's `CALL` and the frame
JIT's call helper both take it.

Probes: `sum(tree)` 34.3M -> 29.4M instructions per 4096-node walk,
`next(g, 0)` over a delegating generator 3469 -> 2179 per item (CPython
1236). Suite, per call: generators_tree 0.945.
---
 crates/weavepy-vm/src/frame_jit.rs            |  20 +++
 crates/weavepy-vm/src/lib.rs                  |  75 +++++++-
 tests/regrtest/test_next_default_inline.py    | 165 ++++++++++++++++++
 tests/regrtest/test_sum_instance_generator.py | 112 ++++++++++++
 4 files changed, 369 insertions(+), 3 deletions(-)
 create mode 100644 tests/regrtest/test_next_default_inline.py
 create mode 100644 tests/regrtest/test_sum_instance_generator.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 2bd63b37..e12890e3 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -4673,6 +4673,26 @@ unsafe extern "C" fn h_call(
             }
             return RELOAD;
         }
+        // `next(gen, default)`: likewise, the default the result of an
+        // exhausted generator.
+        if argc == 2
+            && matches!(
+                (&*base.add(start), &*base.add(start + 1), &*base.add(start + 2)),
+                (Object::Builtin(b), Object::Unbound, Object::Generator(_))
+                    if Rc::as_ptr(b) as usize == interp.leaf_fns().next_ptr
+            )
+        {
+            if st.sw.is_null() {
+                return CALL_DECLINED;
+            }
+            sync(st);
+            let sw = &mut *st.sw;
+            let interp = &mut *st.interp.cast_mut();
+            if !interp.core_gen_resume(sw, pc, crate::InlineResume::NextDefault) {
+                sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+            }
+            return RELOAD;
+        }
         Interpreter::core_instance_callee(base.add(start));
         let done = |st: &mut State, r: Object| {
             for k in start..len {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index f329fc3b..a5fce078 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -8308,6 +8308,7 @@ impl Interpreter {
     ) -> Option<Box<InlineAct>> {
         let arg = match mode {
             InlineResume::NextCall | InlineResume::SendCall => GEN_NEXT_CALL,
+            InlineResume::NextDefault => GEN_NEXT_DEFAULT,
             InlineResume::ForIter => frame.code.instructions.get(pc)?.arg,
             InlineResume::Send => {
                 let jump = frame.code.instructions.get(pc)?.arg;
@@ -8352,6 +8353,12 @@ impl Interpreter {
                 }
                 _ => return None,
             }
+        } else if mode == InlineResume::NextDefault {
+            // `[next, NULL, gen, default]`.
+            let Some(Object::Generator(g)) = frame.stack.get(n.checked_sub(2)?) else {
+                return None;
+            };
+            (g, true)
         } else {
             let Some(Object::Generator(g)) = frame.stack.last() else {
                 return None;
@@ -8428,6 +8435,14 @@ impl Interpreter {
                 let (a, b, c) = (frame.stack.pop(), frame.stack.pop(), frame.stack.pop());
                 drop((a, b, c));
             }
+            // Likewise, the default staying on top (see
+            // [`GEN_NEXT_DEFAULT`]).
+            InlineResume::NextDefault => {
+                let default = frame.stack.pop().expect("checked above");
+                let (a, b, c) = (frame.stack.pop(), frame.stack.pop(), frame.stack.pop());
+                frame.stack.push(default);
+                drop((a, b, c));
+            }
             // The bound `send` and its empty self slot, or the method and
             // the generator (the value went in).
             InlineResume::SendCall => {
@@ -8567,6 +8582,10 @@ impl Interpreter {
         }
         match result {
             Ok(GenStep::Yielded(v)) => {
+                // `next(gen, default)`: the value takes the default's place.
+                if arg == GEN_NEXT_DEFAULT {
+                    drop(frame.stack.pop());
+                }
                 frame.stack.push(v);
                 QuietEntry::Returned { cur_pc: call_pc }
             }
@@ -8575,6 +8594,11 @@ impl Interpreter {
                 err: crate::error::stop_iteration_with(v),
                 cur_pc: call_pc,
             },
+            // `next(gen, default)`: the default is the result.
+            Ok(GenStep::Exhausted(v)) if arg == GEN_NEXT_DEFAULT => {
+                drop(v);
+                QuietEntry::Returned { cur_pc: call_pc }
+            }
             // `SEND`: the return value is the `yield from`/`await` result;
             // the sub-generator stays for `END_SEND`.
             Ok(GenStep::Exhausted(v)) if arg & GEN_SEND != 0 => {
@@ -13561,6 +13585,26 @@ impl Interpreter {
                             }
                             break Some(CoreExit::Reload);
                         }
+                        // `next(gen, default)`: as `next(gen)`, with the
+                        // default the result of an exhausted generator.
+                        // SAFETY: `len >= argc + 2 == 4`.
+                        if argc == 2
+                            && !plain_fn
+                            && matches!(
+                                unsafe { (&*base.add(len - 4), &*base.add(len - 3), &*base.add(len - 2)) },
+                                (Object::Builtin(b), Object::Unbound, Object::Generator(_))
+                                    if Rc::as_ptr(b) as usize == self.leaf_fns().next_ptr
+                            )
+                        {
+                            // SAFETY: `len <= cap`, every slot initialized.
+                            unsafe { frame.stack.set_len(len) };
+                            frame.pc = pc as u32;
+                            *last_pc = last;
+                            if !self.core_gen_resume(sw, pc, InlineResume::NextDefault) {
+                                sw.pending = Some(CoreExit::Stop(LeafStop::Step));
+                            }
+                            break Some(CoreExit::Reload);
+                        }
                         // SAFETY: `len >= argc + 2`: the callee and its self
                         // slot.
                         unsafe { Self::core_instance_callee(base.add(len - argc - 2)) };
@@ -20812,6 +20856,7 @@ impl Interpreter {
                 self.core_uncollapse(&mut done);
             }
             let call_pc = done.call_pc;
+            let next_default = done.exhaust_arg == GEN_NEXT_DEFAULT;
             self.lean_pending_exit(done.caller_pending);
             // SAFETY: the shell was taken above: no drop glue owed.
             unsafe { std::ptr::write(&raw mut done.act.shell, None) };
@@ -20823,6 +20868,11 @@ impl Interpreter {
             let mut tmp = None;
             // SAFETY: the consumer is the innermost remaining activation.
             let (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) };
+            // `next(gen, default)`: the value takes the default's place.
+            if next_default {
+                // SAFETY: as above.
+                drop(unsafe { (*cframe).stack.pop() });
+            }
             sw.cur = cframe;
             sw.last = if clast == &raw mut sw.scratch {
                 sw.scratch = usize::MAX;
@@ -20942,7 +20992,11 @@ impl Interpreter {
         let (cframe, clast, cshell) = unsafe { sw.activation(inl.len(), &mut tmp) };
         // SAFETY: as above.
         let consumer = unsafe { &mut *cframe };
-        if arg & GEN_SEND != 0 {
+        if arg == GEN_NEXT_DEFAULT {
+            // `next(gen, default)`: the default (atop the consumer's stack)
+            // is the result.
+            drop(v);
+        } else if arg & GEN_SEND != 0 {
             // `SEND`: the return value is the `yield from`/`await` result;
             // the sub-generator stays for `END_SEND`.
             consumer.stack.push(v);
@@ -38566,7 +38620,13 @@ impl Interpreter {
         // instead of materializing the whole sequence first; CPython's
         // `builtin_sum_impl` is exactly this drive loop.
         let mut total = SumState::new(acc);
-        if let Object::Generator(g) = &args[0] {
+        // (An iterable whose `__iter__` is a generator function, as a tree
+        // walked by `yield from`, drains the same way.)
+        let iter = match &args[0] {
+            Object::Generator(_) => args[0].clone(),
+            other => self.make_iter(other, globals)?,
+        };
+        if let Object::Generator(g) = &iter {
             loop {
                 // A lean resume folds scalar yields straight into `total`
                 // (see `Interpreter::sum_fold`); it returns at the first
@@ -38581,7 +38641,6 @@ impl Interpreter {
         }
         // Stream the live iterator. Addition can mutate the source or
         // raise, so don't consume later elements or request a length hint.
-        let iter = self.make_iter(&args[0], globals)?;
         while let Some(x) = self.iter_next(&iter, globals)? {
             total.add(self, x)?;
         }
@@ -65268,6 +65327,12 @@ const GEN_NEXT_CALL: u32 = u32::MAX;
 /// its return value on the delegator's stack and takes the jump.
 const GEN_SEND: u32 = 1 << 31;
 
+/// [`InlineAct::exhaust_arg`] of a generator resumed by `next(gen,
+/// default)`: the default waits atop the caller's stack, replaced by a
+/// yielded value, or left as the call's result when the generator
+/// finishes. (Checked before [`GEN_SEND`], whose bit it has.)
+const GEN_NEXT_DEFAULT: u32 = u32::MAX - 1;
+
 /// How [`Interpreter::core_send_hop`] went.
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum SendHop {
@@ -65288,6 +65353,10 @@ enum InlineResume {
     ForIter,
     /// A `CALL` of builtin `next` on the generator at the top of the stack.
     NextCall,
+    /// A `CALL` of builtin `next` on a generator with a default (the top
+    /// of the stack): as [`Self::NextCall`], but exhaustion gives the
+    /// default.
+    NextDefault,
     /// `SEND` of the value at the top of the stack to the generator or
     /// coroutine below it (`yield from`, `await`).
     Send,
diff --git a/tests/regrtest/test_next_default_inline.py b/tests/regrtest/test_next_default_inline.py
new file mode 100644
index 00000000..b564279d
--- /dev/null
+++ b/tests/regrtest/test_next_default_inline.py
@@ -0,0 +1,165 @@
+"""`next(gen, default)` resumes the generator inline, as `next(gen)` does,
+with the default waiting atop the caller's stack until the generator
+yields (the value replaces it) or finishes (it is the result); see
+`InlineResume::NextDefault` in the VM. Each case runs warm (loops long
+enough to compile the caller) and compares with the general path.
+"""
+
+import sys
+import unittest
+
+WARM = 3000
+
+
+def count(n):
+    for i in range(n):
+        yield i
+
+
+def returns(n):
+    yield n
+    return "ignored"
+
+
+def raises():
+    yield 1
+    raise KeyError("inner")
+
+
+def leaky():
+    yield 1
+    raise StopIteration
+
+
+class Tree:
+    __slots__ = ("left", "value", "right")
+
+    def __init__(self, left, value, right):
+        self.left = left
+        self.value = value
+        self.right = right
+
+    def __iter__(self):
+        if self.left:
+            yield from self.left
+        yield self.value
+        if self.right:
+            yield from self.right
+
+
+def build(lo, hi):
+    if lo >= hi:
+        return None
+    mid = (lo + hi) // 2
+    return Tree(build(lo, mid), mid, build(mid + 1, hi))
+
+
+def pairs(it):
+    it = iter(it)
+    for a in it:
+        b = next(it, -1)
+        yield a, b
+
+
+def relay(it, default):
+    while True:
+        v = yield next(it, default)
+        if v is not None:
+            return v
+
+
+class Mortal:
+    deaths = 0
+
+    def __del__(self):
+        Mortal.deaths += 1
+
+
+class NextDefaultTests(unittest.TestCase):
+    def test_values_then_default_warm(self):
+        for n in range(WARM // 100):
+            g = count(n)
+            got = [next(g, None) for _ in range(n + 2)]
+            self.assertEqual(got, list(range(n)) + [None, None])
+
+    def test_loop_warm(self):
+        total = 0
+        g = count(WARM)
+        while (v := next(g, -1)) != -1:
+            total += v
+        self.assertEqual(total, sum(range(WARM)))
+        self.assertEqual(next(g, "done"), "done")
+
+    def test_return_value_ignored(self):
+        for _ in range(WARM):
+            g = returns(3)
+            self.assertEqual(next(g, 0), 3)
+            self.assertEqual(next(g, 0), 0)
+            self.assertEqual(next(g, 0), 0)
+
+    def test_raise_propagates(self):
+        for _ in range(WARM // 10):
+            g = raises()
+            self.assertEqual(next(g, 0), 1)
+            with self.assertRaises(KeyError):
+                next(g, 0)
+            self.assertEqual(next(g, "after"), "after")
+
+    def test_pep479(self):
+        g = leaky()
+        next(g, 0)
+        with self.assertRaises(RuntimeError):
+            next(g, 0)
+
+    def test_tree_pairs(self):
+        tree = build(0, 101)
+        for _ in range(30):
+            got = list(pairs(tree))
+        expected = [(a, a + 1) for a in range(0, 100, 2)] + [(100, -1)]
+        self.assertEqual(got, expected)
+
+    def test_yield_of_next(self):
+        for _ in range(WARM // 10):
+            r = relay(count(3), "end")
+            self.assertEqual([next(r) for _ in range(5)], [0, 1, 2, "end", "end"])
+            with self.assertRaises(StopIteration) as cm:
+                r.send(7)
+            self.assertEqual(cm.exception.value, 7)
+
+    def test_default_released(self):
+        for _ in range(WARM // 10):
+            g = count(1)
+            next(g, Mortal())
+        Mortal.deaths = 0
+        g = count(1)
+        self.assertEqual(next(g, Mortal()), 0)
+        self.assertEqual(Mortal.deaths, 1)
+        self.assertIsInstance(next(g, Mortal()), Mortal)
+        self.assertEqual(Mortal.deaths, 2)
+
+    def test_running_generator(self):
+        def selfish():
+            yield next(me, "default")
+
+        me = selfish()
+        with self.assertRaises(ValueError):
+            next(me, None)
+
+    def test_unstarted_and_closed(self):
+        g = count(2)
+        g.close()
+        self.assertEqual(next(g, "closed"), "closed")
+        g = count(2)
+        self.assertEqual(next(g, "x"), 0)
+
+    def test_frame_line_during_resume(self):
+        def probe():
+            yield sys._getframe(1).f_lineno
+
+        for _ in range(WARM // 10):
+            line = next(probe(), None); expected = sys._getframe().f_lineno
+        self.assertEqual(line, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/regrtest/test_sum_instance_generator.py b/tests/regrtest/test_sum_instance_generator.py
new file mode 100644
index 00000000..a71c1ced
--- /dev/null
+++ b/tests/regrtest/test_sum_instance_generator.py
@@ -0,0 +1,112 @@
+"""`sum(obj)` where `iter(obj)` is a generator (a class whose `__iter__` is
+a generator function, such as a tree walked with `yield from`) drains the
+generator the way `sum(generator)` does, folding the yields it can in
+place (see `do_sum_call` in the VM). Each case checks what plain
+iteration gives: the total, `start`, mixed and float items, a raise
+partway, PEP 479, and that `__iter__` runs exactly once.
+"""
+
+import unittest
+
+WARM = 300
+
+
+class Tree:
+    __slots__ = ("left", "value", "right")
+
+    def __init__(self, left, value, right):
+        self.left = left
+        self.value = value
+        self.right = right
+
+    def __iter__(self):
+        if self.left:
+            yield from self.left
+        yield self.value
+        if self.right:
+            yield from self.right
+
+
+def build(lo, hi):
+    if lo >= hi:
+        return None
+    mid = (lo + hi) // 2
+    return Tree(build(lo, mid), mid, build(mid + 1, hi))
+
+
+class Items:
+    def __init__(self, items):
+        self.items = items
+        self.iters = 0
+
+    def __iter__(self):
+        self.iters += 1
+        for x in self.items:
+            yield x
+
+
+class Failing:
+    def __iter__(self):
+        yield 1
+        yield 2
+        raise KeyError("boom")
+
+
+class LeakyStop:
+    def __iter__(self):
+        yield 1
+        raise StopIteration
+
+
+class Returning:
+    def __iter__(self):
+        yield 5
+        return 99
+
+
+class SumInstanceGeneratorTests(unittest.TestCase):
+    def test_tree_warm(self):
+        tree = build(0, 256)
+        for _ in range(WARM):
+            self.assertEqual(sum(tree), sum(range(256)))
+        self.assertEqual(sum(tree, 1000), sum(range(256)) + 1000)
+        self.assertEqual(sum(tree, start=-1), sum(range(256)) - 1)
+
+    def test_iter_called_once(self):
+        items = Items([1, 2, 3])
+        self.assertEqual(sum(items), 6)
+        self.assertEqual(items.iters, 1)
+
+    def test_mixed_items(self):
+        self.assertEqual(sum(Items([1, 2.5, True, 10**30])), 1 + 2.5 + 1 + 10**30)
+        self.assertEqual(sum(Items([0.1] * 10)), sum([0.1] * 10))
+        self.assertEqual(sum(Items([[1], [2]]), []), [1, 2])
+        with self.assertRaises(TypeError):
+            sum(Items([1, "a"]))
+
+    def test_raise_partway(self):
+        with self.assertRaises(KeyError):
+            sum(Failing())
+
+    def test_pep479(self):
+        with self.assertRaises(RuntimeError) as cm:
+            sum(LeakyStop())
+        self.assertIsInstance(cm.exception.__cause__, StopIteration)
+
+    def test_return_value_ignored(self):
+        self.assertEqual(sum(Returning()), 5)
+
+    def test_non_generator_iter(self):
+        class ListIter:
+            def __iter__(self):
+                return iter([4, 5, 6])
+
+        self.assertEqual(sum(ListIter()), 15)
+
+    def test_empty(self):
+        self.assertEqual(sum(Items([])), 0)
+        self.assertEqual(sum(Items([]), 7), 7)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 880d5c74081e194663825f24175f84773049b40e Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 07:50:53 -0700
Subject: [PATCH 054/173] perf: raise next(gen)'s StopIteration from the core
 loop

A generator resumed inline by `next(gen)` or `gen.send(v)` that returned
left the core loop for `quiet_run`, which finished the activation and
raised `StopIteration(value)` at the call. `core_gen_return` now
finishes it like any other inline generator return and leaves the raise
pending at the call (the leaf raise shape), which the quiet loop's
handler search takes as it took the delivered one.

`next(gen)` raising `StopIteration` into an `except` clause in the same
frame (`_GeneratorContextManager.__exit__`): 2504M -> 2442M
instructions over 300k iterations. The suite effect is within noise.
---
 crates/weavepy-vm/src/lib.rs | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index a5fce078..a2b5fb8a 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -20945,10 +20945,8 @@ impl Interpreter {
         let Some(top) = inl.last_mut() else {
             return false;
         };
-        // A shell (the general epilogue's), or `next(gen)`'s exhaustion
-        // (a `StopIteration`), finishes through `quiet_run`.
+        // A shell (the general epilogue's) finishes through `quiet_run`.
         if top.act.shell.is_some()
-            || top.exhaust_arg == GEN_NEXT_CALL
             || self.countdown_out(2)
             || crate::hot_gates::loop_gen() != snap_gen
         {
@@ -20992,7 +20990,13 @@ impl Interpreter {
         let (cframe, clast, cshell) = unsafe { sw.activation(inl.len(), &mut tmp) };
         // SAFETY: as above.
         let consumer = unsafe { &mut *cframe };
-        if arg == GEN_NEXT_DEFAULT {
+        let mut raised = None;
+        if arg == GEN_NEXT_CALL {
+            // `next(gen)` or `gen.send(v)`: `StopIteration(value)`, raised
+            // at the call (the consumer's `pc` is past it), as
+            // `inline_gen_deliver` raises it.
+            raised = Some(crate::error::stop_iteration_with(v));
+        } else if arg == GEN_NEXT_DEFAULT {
             // `next(gen, default)`: the default (atop the consumer's stack)
             // is the result.
             drop(v);
@@ -21016,6 +21020,11 @@ impl Interpreter {
         } else {
             clast
         };
+        // The raise protocol of a call that raised (the leaf raise shape).
+        if let Some(e) = raised {
+            sw.pending = Some(CoreExit::Stop(LeafStop::Raised(e)));
+            return true;
+        }
         // The consumer's `QuietEntry::Returned` protocol (`quiet_frame`).
         // SAFETY: as above.
         unsafe { *sw.last = call_pc };

From 740b19990b9c63ed0513feeb7bf64095bb7c2be1 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 08:26:07 -0700
Subject: [PATCH 055/173] perf: keep ExitStack's per-use steps out of the slow
 path

Four steps `contextlib.ExitStack` takes on every use ran through the
full handlers:

- `_exit_wrapper.__wrapped__ = callback`: the core loop stored a
  function's own attributes in its `__dict__` only for names without a
  leading `__`. It now declines just the names the function type and
  `object` define as data descriptors, which the full handler validates
  or routes to the slots.
- `callback(*args, **kwds)` of a bound builtin (`log.clear`):
  `CALL_FUNCTION_EX` over a tuple with no keywords takes the builtin
  lane `CALL` takes, through `core_call_lane`, for the builtins that
  lane admits.
- `MethodType(cm_exit, cm)`: the `method` type's two-argument
  construction joins the constructors the core loop runs in place,
  binding as the full constructor does.
- `sys.exception()` reads the handled-exception stack and runs no
  Python code: it's registered as a leaf builtin.

`with ExitStack()` entering one manager and one callback: 7658M ->
6610M instructions over 100k uses. Suite: context_managers 0.975.
---
 crates/weavepy-vm/src/lib.rs                |  37 ++++-
 crates/weavepy-vm/src/seqiter.rs            |  10 ++
 crates/weavepy-vm/src/stdlib/sys.rs         |  28 ++--
 tests/regrtest/test_exitstack_core_paths.py | 156 ++++++++++++++++++++
 4 files changed, 215 insertions(+), 16 deletions(-)
 create mode 100644 tests/regrtest/test_exitstack_core_paths.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index a2b5fb8a..3edad930 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -16027,7 +16027,8 @@ impl Interpreter {
     /// `CALL_KW` or `CALL_FUNCTION_EX` (an exact tuple, and no `**` or an
     /// exact dict of `str` keys) at the running activation's pc, of a
     /// class or a Python function the inline paths declined (a class with
-    /// its own `__new__`, a callee binding `*args`): the call runs through
+    /// its own `__new__`, a callee binding `*args`), or of a builtin the
+    /// builtin lane takes, spread without keywords: the call runs through
     /// the ordinary protocol with this activation published, as the
     /// builtin lane runs a builtin. Zero-argument `super` (which reads its
     /// caller's frame) stays the full handler's. `false` touches nothing.
@@ -16061,11 +16062,26 @@ impl Interpreter {
             }
             _ => return false,
         };
+        // (A builtin the `CALL` arm's builtin lane takes, spread from a
+        // tuple without keywords: `callback(*args, **kwds)` of a bound
+        // `list.clear`.)
+        let mut builtin = false;
         let ok = match &frame.stack[callee_at] {
             Object::Type(t) => !Rc::ptr_eq(t, &builtin_types().super_),
             Object::Function(_) => true,
-            Object::BoundMethod(bm) => {
-                !bm.redispatch_descriptor && matches!(bm.function, Object::Function(_))
+            Object::BoundMethod(bm) if !bm.redispatch_descriptor => match &bm.function {
+                Object::Function(_) => true,
+                Object::Builtin(b) if ins.op == OpCode::CallEx => {
+                    builtin = true;
+                    matches!(frame.stack[callee_at + 1], Object::Unbound)
+                        && self.builtin_lane(b, Some(&bm.receiver)).is_some()
+                }
+                _ => false,
+            },
+            Object::Builtin(b) if ins.op == OpCode::CallEx => {
+                builtin = true;
+                matches!(frame.stack[callee_at + 1], Object::Unbound)
+                    && self.builtin_lane(b, None).is_some()
             }
             _ => false,
         };
@@ -16112,6 +16128,9 @@ impl Interpreter {
                 _ => return false,
             }
         }
+        if builtin && !kwargs.is_empty() {
+            return false;
+        }
         // Committed: the operands leave the stack.
         let callee = frame.stack[callee_at].clone();
         let operands: Vec<Object> = frame.stack.drain(callee_at..).collect();
@@ -21646,7 +21665,17 @@ impl Interpreter {
         let Some(name @ Object::Str(n)) = code_name_obj(code, ins.arg) else {
             return None;
         };
-        if n.starts_with("__") {
+        // The names the function type (and `object`) define as data
+        // descriptors are the full handler's; any other lands in the
+        // function's `__dict__` (`wrapper.__wrapped__ = f`).
+        if n.starts_with("__")
+            && (crate::object::is_function_slot(n)
+                || matches!(
+                    &**n,
+                    "__code__" | "__globals__" | "__closure__" | "__builtins__" | "__dict__"
+                        | "__class__"
+                ))
+        {
             return None;
         }
         let n_ = stack.len();
diff --git a/crates/weavepy-vm/src/seqiter.rs b/crates/weavepy-vm/src/seqiter.rs
index 6b82238e..cb84c2fa 100644
--- a/crates/weavepy-vm/src/seqiter.rs
+++ b/crates/weavepy-vm/src/seqiter.rs
@@ -1271,6 +1271,16 @@ pub(crate) fn builtin_ctor_pure(cls: &Rc<TypeObject>, args: &[Object]) -> Option
             _ => None,
         };
     }
+    // `types.MethodType(func, obj)`: `func` bound to `obj`, as the full
+    // constructor binds it.
+    if Rc::ptr_eq(cls, &bt.method_) {
+        return match args {
+            [func, obj] => Some(Object::BoundMethod(Rc::new(
+                crate::object::BoundMethod::py_method(obj.clone(), func.clone()),
+            ))),
+            _ => None,
+        };
+    }
     // `set()`, `dict()`, `list()`, `tuple()`: an empty one (a container
     // tracked like the builtins' own, unless that would start a collection).
     if args.is_empty() {
diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs
index 2e913f34..049a5220 100644
--- a/crates/weavepy-vm/src/stdlib/sys.rs
+++ b/crates/weavepy-vm/src/stdlib/sys.rs
@@ -96,20 +96,24 @@ pub fn build_with_state(
             })),
         );
         let es_fallback_exc = exc_info_stack.clone();
+        let exception = Rc::new(BuiltinFn {
+            name: "exception",
+            binds_instance: false,
+            call: Box::new(move |_| {
+                if let Some(h) = crate::vm_singletons::current_thread_handles() {
+                    sys_exception(&h.exc_info_stack)
+                } else {
+                    sys_exception(&es_fallback_exc)
+                }
+            }),
+            call_kw: None,
+        });
+        // Reads the handled-exception stack: no Python runs, and no frame
+        // is looked at (`ExitStack.__exit__` calls it every time).
+        crate::leaf_builtins::register(&exception);
         d.insert(
             DictKey(Object::from_static("exception")),
-            Object::Builtin(Rc::new(BuiltinFn {
-                name: "exception",
-                binds_instance: false,
-                call: Box::new(move |_| {
-                    if let Some(h) = crate::vm_singletons::current_thread_handles() {
-                        sys_exception(&h.exc_info_stack)
-                    } else {
-                        sys_exception(&es_fallback_exc)
-                    }
-                }),
-                call_kw: None,
-            })),
+            Object::Builtin(exception),
         );
         d.insert(
             DictKey(Object::from_static("__excepthook__")),
diff --git a/tests/regrtest/test_exitstack_core_paths.py b/tests/regrtest/test_exitstack_core_paths.py
new file mode 100644
index 00000000..2f7c22df
--- /dev/null
+++ b/tests/regrtest/test_exitstack_core_paths.py
@@ -0,0 +1,156 @@
+"""The steps `contextlib.ExitStack` takes on every use, run in the core
+loop instead of as slow steps: a function's own dunder attribute store
+(`_exit_wrapper.__wrapped__ = callback`), `callback(*args, **kwds)` of a
+bound builtin method, `types.MethodType(cm_exit, cm)`, and
+`sys.exception()`. Each case runs warm and checks the observable
+results against what the general paths give.
+"""
+
+import contextlib
+import sys
+import types
+import unittest
+
+WARM = 2000
+
+
+class Manager:
+    def __init__(self, log):
+        self.log = log
+
+    def __enter__(self):
+        self.log.append("enter")
+        return self
+
+    def __exit__(self, *exc):
+        self.log.append(("exit", exc[0]))
+        return False
+
+
+class ExitStackCorePathTests(unittest.TestCase):
+    def test_exitstack_warm(self):
+        log = []
+        for i in range(WARM):
+            with contextlib.ExitStack() as stack:
+                m = stack.enter_context(Manager(log))
+                stack.callback(log.append, i)
+                stack.callback(log.clear)
+                self.assertIs(m.log, log)
+            # (Callbacks run last in, first out.)
+            self.assertEqual(log, [i, ("exit", None)])
+            log.clear()
+
+    def test_exitstack_exception(self):
+        log = []
+        for _ in range(WARM // 10):
+            with self.assertRaises(KeyError):
+                with contextlib.ExitStack() as stack:
+                    stack.enter_context(Manager(log))
+                    stack.callback(log.append, "cb")
+                    raise KeyError
+            self.assertEqual(log, ["enter", "cb", ("exit", KeyError)])
+            log.clear()
+
+    def test_function_dunder_attributes(self):
+        def target():
+            pass
+
+        for i in range(WARM):
+            def wrapper():
+                pass
+
+            wrapper.__wrapped__ = target
+            wrapper.__isabstractmethod__ = bool(i & 1)
+            wrapper.plain = i
+        self.assertIs(wrapper.__wrapped__, target)
+        self.assertEqual(
+            wrapper.__dict__,
+            {"__wrapped__": target, "__isabstractmethod__": True, "plain": WARM - 1},
+        )
+        for _ in range(WARM):
+            wrapper.__doc__ = "doc"
+            wrapper.__name__ = "renamed"
+            wrapper.__qualname__ = "Q.renamed"
+            wrapper.__module__ = "mod"
+        self.assertEqual(
+            (wrapper.__doc__, wrapper.__name__, wrapper.__qualname__, wrapper.__module__),
+            ("doc", "renamed", "Q.renamed", "mod"),
+        )
+        self.assertNotIn("__doc__", wrapper.__dict__)
+        self.assertNotIn("__name__", wrapper.__dict__)
+        with self.assertRaises(TypeError):
+            wrapper.__name__ = 1
+        with self.assertRaises(TypeError):
+            wrapper.__dict__ = 1
+        with self.assertRaises(AttributeError):
+            wrapper.__globals__ = {}
+        d = {}
+        wrapper.__dict__ = d
+        wrapper.__wrapped__ = 5
+        self.assertEqual(d, {"__wrapped__": 5})
+
+    def test_call_ex_bound_builtin(self):
+        log = [1, 2, 3]
+        for i in range(WARM):
+            args = (i,)
+            kwds = {}
+            app = log.append
+            app(*args, **kwds)
+            clear = log.clear
+            if i % 100 == 49:
+                clear(*(), **{})
+        self.assertEqual(log, list(range(WARM - 50, WARM)))
+        d = {}
+        upd = d.update
+        for i in range(WARM):
+            upd(*({"k": i},), **{})
+            upd(*(), **{"j": i})
+        self.assertEqual(d, {"k": WARM - 1, "j": WARM - 1})
+        with self.assertRaises(TypeError):
+            log.append(*(1, 2), **{})
+        with self.assertRaises(TypeError):
+            log.append(*(1,), **{"x": 1})
+
+    def test_method_type(self):
+        class C:
+            pass
+
+        def f(self, x):
+            return (self, x)
+
+        obj = C()
+        for i in range(WARM):
+            m = types.MethodType(f, obj)
+        self.assertIs(m.__self__, obj)
+        self.assertIs(m.__func__, f)
+        self.assertEqual(m(4), (obj, 4))
+        with self.assertRaises(TypeError):
+            types.MethodType(f)
+
+    def test_sys_exception(self):
+        self.assertIsNone(sys.exception())
+        seen = []
+        for i in range(WARM):
+            try:
+                raise ValueError(i)
+            except ValueError:
+                seen.append(sys.exception())
+                self.assertIs(sys.exc_info()[1], seen[-1])
+            self.assertIsNone(sys.exception())
+        self.assertEqual(seen[-1].args, (WARM - 1,))
+
+        def gen():
+            try:
+                raise KeyError("g")
+            except KeyError:
+                yield sys.exception()
+                yield sys.exception()
+
+        g = gen()
+        first = next(g)
+        self.assertIsNone(sys.exception())
+        self.assertIs(next(g), first)
+
+
+if __name__ == "__main__":
+    unittest.main()

From f9dcae1dea6d580d48e51e400b02b1be768fbcc3 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 08:58:58 -0700
Subject: [PATCH 056/173] perf: await native asyncio futures in the core loop

`await fut` of a native `Future` (or a subclass keeping its `__await__`)
took two slow steps: `GET_AWAITABLE` called `__await__` through the full
handler, and each `SEND` to the `FutureIter` went through the general
send, the finished future's result riding a `StopIteration`. The core
loop's helper now makes the `FutureIter` in place for a class whose
`__await__` resolves to the native one (remembered per class version),
and the `SEND` arm steps the iterator directly: a pending future is
yielded to the task, a future that finished with a result gives its
result as the `SEND` exits (CPython's `am_send`), and anything else
(an exception, a cancellation, a second step while pending) still takes
the general path.

`loop.create_future()`'s and `call_soon`'s native lanes also remember
where they last found `Future`, `Task` and `Handle` in the asyncio
modules, keyed by the two dicts' mutation stamps and the global value
epoch, instead of probing both dicts each time. `current_task` joins the
builtins the core loop's builtin lane takes (`gather` calls it).

Awaiting a future set by `call_soon`: 89761 -> 82265 instructions per
await (CPython 53842). Suite: async_tree 0.973.
---
 crates/weavepy-vm/src/lib.rs                  |  34 +++++-
 .../weavepy-vm/src/stdlib/asyncio_events.rs   |  82 +++++++++++---
 crates/weavepy-vm/src/stdlib/asyncio_mod.rs   | 103 +++++++++++++++--
 .../test_asyncio_future_await_core.py         | 104 ++++++++++++++++++
 4 files changed, 301 insertions(+), 22 deletions(-)
 create mode 100644 tests/regrtest/test_asyncio_future_await_core.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 3edad930..9848fb6b 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -11000,7 +11000,13 @@ impl Interpreter {
             } else {
                 "__iter__"
             };
-            let Some(gen) = self.instance_gen_call(v, name) else {
+            // (`await fut` of a native asyncio future: its `FutureIter`.)
+            let made = if ins.op == OpCode::GetAwaitable {
+                crate::stdlib::asyncio_mod::native_future_await(v)
+            } else {
+                None
+            };
+            let Some(gen) = made.or_else(|| self.instance_gen_call(v, name)) else {
                 return CoreAttr::Decline;
             };
             let top = frame.stack.last_mut().expect("checked above");
@@ -12625,6 +12631,30 @@ impl Interpreter {
                                 Object::Generator(_) | Object::Coroutine(_)
                             )
                         {
+                            // `await fut` of a native asyncio future: its
+                            // iterator's step, the sent value replaced by
+                            // what it yields, or by the result as the
+                            // `SEND` exits (see `future_iter_send`).
+                            // SAFETY: `len >= 2` (checked first).
+                            let step = (len >= 2 && ins.arg & GEN_SEND == 0)
+                                .then(|| unsafe { &*base.add(len - 2) })
+                                .and_then(crate::stdlib::asyncio_mod::future_iter_send);
+                            if let Some(step) = step {
+                                use crate::stdlib::asyncio_mod::FutureIterStep;
+                                let (v, exit) = match step {
+                                    FutureIterStep::Yield(v) => (v, false),
+                                    FutureIterStep::Return(v) => (v, true),
+                                };
+                                // SAFETY: `len >= 2`: the sent value's slot.
+                                unsafe { drop_hot(std::mem::replace(&mut *base.add(len - 1), v)) };
+                                last = pc;
+                                pc += 1;
+                                if exit {
+                                    pc += ins.arg as usize;
+                                }
+                                after_release!();
+                                continue;
+                            }
                             break Some(CoreExit::Stop(LeafStop::Step));
                         }
                         // SAFETY: `len <= cap`, every slot initialized.
@@ -71385,6 +71415,8 @@ fn builtin_fn_lane_ok(name: &str) -> bool {
             | "_getframe" | "_getframemodulename"
             // warnings (which reads the whole stack, lean activations included)
             | "warn"
+            // asyncio (`gather` reads the running loop's current task)
+            | "current_task"
     )
 }
 
diff --git a/crates/weavepy-vm/src/stdlib/asyncio_events.rs b/crates/weavepy-vm/src/stdlib/asyncio_events.rs
index 5a5fdd75..3e38abfc 100644
--- a/crates/weavepy-vm/src/stdlib/asyncio_events.rs
+++ b/crates/weavepy-vm/src/stdlib/asyncio_events.rs
@@ -358,8 +358,9 @@ fn fast_call_soon(
     // The class the loop's module would build (`events.Handle`, looked up
     // at call time: it may have been replaced, or come from another
     // import of `asyncio.events` than the one `hs` describes).
-    let current = global_module_attr(&ls.globals, &names().events, &names().handle)
-        .is_some_and(|h| matches!(h, Object::Type(t) if Rc::ptr_eq(&t, &hs.cls)));
+    let current =
+        global_module_type_ptr(&ls.globals, &names().events, &names().handle, &HANDLE_MEMO)
+            == Rc::as_ptr(&hs.cls) as usize;
     if !current {
         return Ok(None);
     }
@@ -477,21 +478,55 @@ fn plain_loop(ls: &LoopState, loop_: &Object) -> bool {
         && inst_attr_unset(li, &n.get_debug)
 }
 
-/// `module.attr`, for the module bound to global `module` in `globals`.
-fn global_module_attr(
+/// Where [`global_module_type_ptr`] last found its class: the globals
+/// dict's address and mutation stamp, the module dict's, the global
+/// value epoch, and the class's address.
+type ModuleAttrMemo = std::cell::Cell<(usize, u64, usize, u64, u64, usize)>;
+
+thread_local! {
+    static HANDLE_MEMO: ModuleAttrMemo = const { std::cell::Cell::new((0, 0, 0, 0, 0, 0)) };
+    static TASK_MEMO: ModuleAttrMemo = const { std::cell::Cell::new((0, 0, 0, 0, 0, 0)) };
+    static FUTURE_MEMO: ModuleAttrMemo = const { std::cell::Cell::new((0, 0, 0, 0, 0, 0)) };
+}
+
+/// The address of the class `module.attr` is, for the module bound to
+/// global `module` in `globals` (`0` for anything else), remembered in
+/// `memo` with the states of the two dicts it read:
+/// while neither changed (every store to a dict draws it a new stamp, and
+/// a global rebound in place moves the value epoch), the module the
+/// globals bind is the same one and binds the same class, so the answer
+/// stands without either probe.
+fn global_module_type_ptr(
     globals: &Rc<RefCell<DictData>>,
     module: &AttrName,
     attr: &AttrName,
-) -> Option<Object> {
-    let m = globals
+    memo: &'static std::thread::LocalKey<ModuleAttrMemo>,
+) -> usize {
+    let epoch = crate::object::global_value_epoch();
+    let gptr = Rc::as_ptr(globals) as usize;
+    let Ok(g) = globals.try_borrow() else {
+        return 0;
+    };
+    let gstamp = g.mutation_stamp();
+    drop(g);
+    let (mg, mgs, mm, mms, me, found) = memo.with(std::cell::Cell::get);
+    if mg == gptr && mgs == gstamp && me == epoch && mm != 0 {
+        // SAFETY: the globals are as they were, so they still bind the
+        // module that owns this dict, which they keep alive.
+        let mdict = unsafe { &*(mm as *const RefCell<DictData>) };
+        if mdict.try_borrow().is_ok_and(|d| d.mutation_stamp() == mms) {
+            return found;
+        }
+    }
+    let Some(Object::Module(m)) = globals
         .borrow()
         .get(&crate::object::StrKeyHashed {
             s: module.name,
             hash: module.hash,
         })
-        .cloned()?;
-    let Object::Module(m) = m else {
-        return None;
+        .cloned()
+    else {
+        return 0;
     };
     let v = m
         .dict
@@ -501,7 +536,24 @@ fn global_module_attr(
             hash: attr.hash,
         })
         .cloned();
-    v
+    let found = match &v {
+        Some(Object::Type(t)) => Rc::as_ptr(t) as usize,
+        _ => 0,
+    };
+    if found != 0 {
+        let mstamp = m.dict.borrow().mutation_stamp();
+        memo.with(|c| {
+            c.set((
+                gptr,
+                gstamp,
+                Rc::as_ptr(&m.dict) as usize,
+                mstamp,
+                epoch,
+                found,
+            ));
+        });
+    }
+    found
 }
 
 /// `BaseEventLoop.create_task(self, coro, **kwargs)`.
@@ -522,8 +574,9 @@ fn loop_create_task_method(
                 inst_attr(li, &names().task_factory),
                 Some(Object::None)
             );
-            let native = global_module_attr(&ls.globals, &names().tasks, &names().task)
-                .is_some_and(|t| crate::stdlib::asyncio_mod::is_task_class(&t));
+            let native =
+                global_module_type_ptr(&ls.globals, &names().tasks, &names().task, &TASK_MEMO)
+                    == crate::stdlib::asyncio_mod::task_class_addr();
             if no_factory && native {
                 return crate::stdlib::asyncio_mod::new_task(loop_, coro.clone(), kwargs);
             }
@@ -535,8 +588,9 @@ fn loop_create_task_method(
 /// `BaseEventLoop.create_future(self)`.
 fn loop_create_future_method(ls: &LoopState, args: &[Object]) -> Result<Object, RuntimeError> {
     if let [loop_] = args {
-        let native = global_module_attr(&ls.globals, &names().futures, &names().future)
-            .is_some_and(|f| crate::stdlib::asyncio_mod::is_future_class(&f));
+        let native =
+            global_module_type_ptr(&ls.globals, &names().futures, &names().future, &FUTURE_MEMO)
+                == crate::stdlib::asyncio_mod::future_class_addr();
         // (Not in debug mode: there the Python method's frame shows in
         // the future's `_source_traceback`.)
         if native && plain_loop(ls, loop_) {
diff --git a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
index 6695b0fc..a3d41e05 100644
--- a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs
@@ -1629,6 +1629,10 @@ fn fi_mark_done(inst: &PyInstance) {
 }
 
 fn future_iter_class() -> Rc<TypeObject> {
+    future_iter_class_ref().clone()
+}
+
+fn future_iter_class_ref() -> &'static Rc<TypeObject> {
     static CELL: std::sync::OnceLock<Rc<TypeObject>> = std::sync::OnceLock::new();
     CELL.get_or_init(|| {
         let bt = crate::builtin_types::builtin_types();
@@ -1660,7 +1664,6 @@ fn future_iter_class() -> Rc<TypeObject> {
         TypeObject::new_user("FutureIter", vec![bt.object_.clone()], dict)
             .expect("FutureIter class must linearise")
     })
-    .clone()
 }
 
 fn future_iter_new(fut: &Object) -> Object {
@@ -1718,6 +1721,92 @@ fn fi_send(args: &[Object]) -> Result<Object, RuntimeError> {
     fi_next(&args[..1])
 }
 
+/// How [`future_iter_send`] stepped a `FutureIter`.
+pub(crate) enum FutureIterStep {
+    /// The pending future, yielded to the task driving the `await`.
+    Yield(Object),
+    /// The finished future's result: the `await`'s value.
+    Return(Object),
+}
+
+/// A `SEND` (of any value, which a `FutureIter` ignores) to `it`, when it
+/// is a native `FutureIter` over a pending future, or over a future that
+/// finished with a result: [`fi_next`]'s step, the result returned without
+/// the `StopIteration` that carries it there (CPython's `am_send`).
+/// `None` for anything else (the general `SEND` raises what's owed),
+/// touching nothing. Runs no Python code.
+pub(crate) fn future_iter_send(it: &Object) -> Option<FutureIterStep> {
+    let Object::Instance(inst) = it else {
+        return None;
+    };
+    if !std::ptr::eq(inst.cls_raw(), Rc::as_ptr(future_iter_class_ref())) {
+        return None;
+    }
+    let fut = fi_future(inst);
+    let Object::Instance(fut_inst) = &fut else {
+        return None;
+    };
+    let st = existing_state_of(fut_inst)?;
+    let status = st.try_borrow().ok()?.status;
+    match status {
+        FutStatus::Pending => {
+            let mut s = st.try_borrow_mut().ok()?;
+            if s.blocking {
+                return None;
+            }
+            s.blocking = true;
+            drop(s);
+            Some(FutureIterStep::Yield(fut))
+        }
+        FutStatus::Finished => {
+            let result = {
+                let mut s = st.try_borrow_mut().ok()?;
+                if !matches!(s.exception(), Object::None) {
+                    return None;
+                }
+                s.log_traceback = false;
+                s.result.clone()
+            };
+            fi_mark_done(inst);
+            Some(FutureIterStep::Return(result))
+        }
+        FutStatus::Cancelled => None,
+    }
+}
+
+/// `await fut`'s iterator, when `obj` is an instance whose class's
+/// `__await__` is the native `Future`'s or `Task`'s (a `FutureIter`, as
+/// [`fut_await`] makes it); `None` for anything else.
+pub(crate) fn native_future_await(obj: &Object) -> Option<Object> {
+    thread_local! {
+        /// The last class found to resolve `__await__` natively, and its
+        /// attribute version then.
+        static NATIVE_AWAIT: std::cell::Cell<(usize, u64)> = const { std::cell::Cell::new((0, 0)) };
+    }
+    let Object::Instance(inst) = obj else {
+        return None;
+    };
+    let cls = inst.cls_raw();
+    let key = (std::ptr::from_ref(cls) as usize, cls.attr_version.get());
+    if NATIVE_AWAIT.with(std::cell::Cell::get) != key {
+        let native = |c: &Rc<TypeObject>| {
+            c.dict
+                .try_borrow()
+                .ok()
+                .and_then(|d| d.get(&crate::object::StrKey("__await__")).cloned())
+        };
+        let found = cls.lookup("__await__");
+        let ok = matches!(&found, Some(Object::Builtin(b)) if [future_class_ref(), task_class_ref()]
+            .into_iter()
+            .any(|c| matches!(native(c), Some(Object::Builtin(n)) if Rc::ptr_eq(&n, b))));
+        if !ok {
+            return None;
+        }
+        NATIVE_AWAIT.with(|c| c.set(key));
+    }
+    Some(future_iter_new(obj))
+}
+
 fn fi_throw(args: &[Object]) -> Result<Object, RuntimeError> {
     let inst = fi_self(args)?;
     let interp = interp()?;
@@ -2648,14 +2737,14 @@ fn install_getset(
         .insert(DictKey(Object::from_static(name)), prop);
 }
 
-/// Whether `obj` is the native `Future` class itself.
-pub(crate) fn is_future_class(obj: &Object) -> bool {
-    matches!(obj, Object::Type(t) if Rc::ptr_eq(t, future_class_ref()))
+/// The native `Task` class's address.
+pub(crate) fn task_class_addr() -> usize {
+    Rc::as_ptr(task_class_ref()) as usize
 }
 
-/// Whether `obj` is the native `Task` class itself.
-pub(crate) fn is_task_class(obj: &Object) -> bool {
-    matches!(obj, Object::Type(t) if Rc::ptr_eq(t, task_class_ref()))
+/// The native `Future` class's address.
+pub(crate) fn future_class_addr() -> usize {
+    Rc::as_ptr(future_class_ref()) as usize
 }
 
 /// A tracked, uninitialized instance of the native class `cls`.
diff --git a/tests/regrtest/test_asyncio_future_await_core.py b/tests/regrtest/test_asyncio_future_await_core.py
new file mode 100644
index 00000000..925b3865
--- /dev/null
+++ b/tests/regrtest/test_asyncio_future_await_core.py
@@ -0,0 +1,104 @@
+"""`await fut` of a native asyncio future in the core loop: `GET_AWAITABLE`
+makes the `FutureIter` in place, and `SEND` steps it without a
+`StopIteration` for a future that finished with a result (see
+`asyncio_mod::future_iter_send` in the VM). Exceptions, cancellation and
+anything else the iterator owes still go through the general path. Each
+case runs warm and checks the results asyncio documents.
+"""
+
+import asyncio
+import unittest
+
+WARM = 500
+
+
+class MyFuture(asyncio.Future):
+    pass
+
+
+class OwnAwait(asyncio.Future):
+    def __await__(self):
+        return (yield "custom")
+
+
+class FutureAwaitCoreTests(unittest.TestCase):
+    def run_main(self, coro):
+        return asyncio.run(coro)
+
+    def test_results_warm(self):
+        async def main():
+            loop = asyncio.get_running_loop()
+            total = 0
+            for i in range(WARM):
+                fut = loop.create_future()
+                loop.call_soon(fut.set_result, i)
+                total += await fut
+            done = loop.create_future()
+            done.set_result(7)
+            return total, await done, await done
+
+        self.assertEqual(self.run_main(main()), (sum(range(WARM)), 7, 7))
+
+    def test_subclass_warm(self):
+        async def main():
+            loop = asyncio.get_running_loop()
+            out = []
+            for i in range(WARM):
+                fut = MyFuture(loop=loop)
+                loop.call_soon(fut.set_result, (i, "x"))
+                out.append(await fut)
+            return out[-1], len(out)
+
+        self.assertEqual(self.run_main(main()), ((WARM - 1, "x"), WARM))
+
+    def test_exception_and_cancel(self):
+        async def main():
+            loop = asyncio.get_running_loop()
+            seen = []
+            for i in range(WARM // 10):
+                fut = loop.create_future()
+                loop.call_soon(fut.set_exception, KeyError(i))
+                try:
+                    await fut
+                except KeyError as e:
+                    seen.append(e.args[0])
+                fut = loop.create_future()
+                loop.call_soon(fut.cancel)
+                try:
+                    await fut
+                except asyncio.CancelledError:
+                    seen.append("cancelled")
+            return seen
+
+        seen = self.run_main(main())
+        self.assertEqual(seen[:4], [0, "cancelled", 1, "cancelled"])
+        self.assertEqual(len(seen), 2 * (WARM // 10))
+
+    def test_own_await(self):
+        def drive(coro):
+            return coro.send(None)
+
+        async def custom():
+            return await OwnAwait(loop=asyncio.new_event_loop())
+
+        self.assertEqual(drive(custom()), "custom")
+
+    def test_gather_and_tasks(self):
+        async def leaf(i):
+            await asyncio.sleep(0)
+            return i
+
+        async def main():
+            total = 0
+            for _ in range(WARM // 10):
+                res = await asyncio.gather(*[leaf(i) for i in range(5)])
+                total += sum(res)
+                t = asyncio.ensure_future(leaf(10))
+                total += await t
+            return total
+
+        self.assertEqual(self.run_main(main()), (10 + 10) * (WARM // 10))
+
+
+if __name__ == "__main__":
+    unittest.main()

From bc7cab468c38a77c7d7f11b546a9a9d0c1f417c1 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 09:58:59 -0700
Subject: [PATCH 057/173] perf: make a compiled caller's generators without the
 call prologue

The frame JIT's call helper tried `core_call` before `core_gen_call` for
every Python callee; `core_call` admits no generator-family code, but
reached that verdict only after its own prologue (the inline-call gates,
unpacking a bound callee, the code fetch). A plain function whose code
is a generator, coroutine or async generator now goes to
`core_gen_call` directly.

`await fib(n - 1)` coroutines: 903M -> 886M instructions over 1500
fib(10) drives (1.9%).
---
 crates/weavepy-vm/src/frame_jit.rs | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index e12890e3..5d37cda0 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -4771,11 +4771,19 @@ unsafe extern "C" fn h_call(
             if st.sw.is_null() {
                 return CALL_DECLINED;
             }
+            // A generator function's call makes its generator in place
+            // (`core_call` admits none).
+            let gen_fn = matches!(&ops[0], Object::Function(f) if {
+                let c = &*f.code.as_ptr();
+                c.is_generator || c.is_coroutine || c.is_async_generator
+            });
             sync(st);
             let sw = &mut *st.sw;
             let saved_last = sw.last;
             let interp = &mut *st.interp.cast_mut();
-            let switched = if python == 1 {
+            let switched = if gen_fn {
+                interp.core_gen_call(sw, pc)
+            } else if python == 1 {
                 interp.core_call(sw, pc) || interp.core_gen_call(sw, pc)
             } else {
                 interp.core_new(sw, pc, st.snap_gen)

From 23155dde5979b816039538f1e0365c70019ac57a Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 04:41:53 -0700
Subject: [PATCH 058/173] perf: cheaper float ratios, type(x), codec error
 args, isinstance misses

float.as_integer_ratio answers in machine words when both terms fit an
i64 (the denominator is a power of two, so lowest terms only strip the
mantissa's trailing zeros); statistics' _exact_ratio runs it per item.

type(x) called as a class from native code (map(type, xs), groupby's
key function) returns the class before instantiate's constructor
dispatch.

str.encode and bytes.decode leaf halves accept one of the standard
error handler names as the second argument: a conversion that meets no
error never consults its handler. The exact spellings of the plain
codecs match without the normalizing copy.

An isinstance miss on an instance whose class overrides neither
__class__ nor __getattribute__ is final; the leaf and the full check no
longer read __class__ through the generic attribute path. The class's
memoized dunder resolution (now with __class__) tracks both.
---
 crates/weavepy-vm/src/builtins.rs             |  59 ++++++++-
 crates/weavepy-vm/src/lib.rs                  |  19 ++-
 crates/weavepy-vm/src/types.rs                |  14 ++-
 .../test_isinstance_class_attr_miss.py        | 117 ++++++++++++++++++
 4 files changed, 203 insertions(+), 6 deletions(-)
 create mode 100644 tests/regrtest/test_isinstance_class_attr_miss.py

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 327896a1..6fbea137 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -6040,6 +6040,33 @@ fn float_as_integer_ratio(args: &[Object]) -> Result<Object, RuntimeError> {
     let sign = if (bits >> 63) & 1 == 1 { -1i32 } else { 1 };
     let exp_field = ((bits >> 52) & 0x7FF) as i32;
     let mantissa_field = bits & ((1u64 << 52) - 1);
+    // Machine-word answer when both terms fit an `i64` (every float of
+    // magnitude below 2**10 with few fraction bits, the statistics
+    // module's common case): the denominator is a power of two, so
+    // lowest terms only strip the mantissa's trailing zeros.
+    {
+        let (m, e) = if exp_field == 0 {
+            (mantissa_field, -1074)
+        } else {
+            ((1u64 << 52) | mantissa_field, exp_field - 1075)
+        };
+        if m == 0 {
+            return Ok(Object::new_tuple_array([Object::Int(0), Object::Int(1)]));
+        }
+        let small = if e >= 0 {
+            // `m < 2**53`, so the shift stays below 2**63 while `e <= 9`.
+            (e <= 9).then(|| (i64::try_from(m << e).ok(), Some(1i64)))
+        } else {
+            let shift = m.trailing_zeros().min(e.unsigned_abs());
+            let m = m >> shift;
+            let e = e + shift as i32;
+            (e > -63).then(|| (i64::try_from(m).ok(), Some(1i64 << -e)))
+        };
+        if let Some((Some(num), Some(den))) = small {
+            let num = if sign < 0 { -num } else { num };
+            return Ok(Object::new_tuple_array([Object::Int(num), Object::Int(den)]));
+        }
+    }
     let (mantissa, exponent): (BigInt, i32) = if exp_field == 0 {
         // Subnormal.
         (BigInt::from(mantissa_field), -1074)
@@ -12624,6 +12651,9 @@ fn str_encode(args: &[Object], kwargs: &[(String, Object)]) -> Result<Object, Ru
                 [_, Object::Str(enc)] => {
                     matches!(&**enc, "utf-8" | "utf8" | "UTF-8" | "UTF8" | "utf_8")
                 }
+                [_, Object::Str(enc), errors] if is_builtin_error_handler(errors) => {
+                    matches!(&**enc, "utf-8" | "utf8" | "UTF-8" | "UTF8" | "utf_8")
+                }
                 _ => false,
             };
         if utf8 {
@@ -13760,6 +13790,13 @@ fn plain_codec(name: Option<&Object>) -> Option<PlainCodec> {
     if n.len() > 10 {
         return None;
     }
+    // The spellings code writes, without the normalizing copy.
+    match n {
+        "utf-8" | "utf8" => return Some(PlainCodec::Utf8),
+        "ascii" => return Some(PlainCodec::Ascii),
+        "latin-1" | "latin1" => return Some(PlainCodec::Latin1),
+        _ => {}
+    }
     let n = n.to_ascii_lowercase().replace('_', "-");
     Some(match n.as_str() {
         "utf-8" | "utf8" => PlainCodec::Utf8,
@@ -13769,13 +13806,31 @@ fn plain_codec(name: Option<&Object>) -> Option<PlainCodec> {
     })
 }
 
+/// One of the codec error handlers every interpreter registers. A
+/// conversion that meets no error never consults its handler, so a leaf
+/// half that only serves error-free conversions may accept any of these
+/// (an unknown name is left to the full path, which owns its checks).
+fn is_builtin_error_handler(o: &Object) -> bool {
+    matches!(o, Object::Str(e) if matches!(
+        &**e,
+        "strict"
+            | "ignore"
+            | "replace"
+            | "surrogateescape"
+            | "surrogatepass"
+            | "backslashreplace"
+            | "xmlcharrefreplace"
+            | "namereplace"
+    ))
+}
+
 /// `s.encode([encoding])` with a plain codec, when every character
 /// encodes (an unencodable one raises on the full path).
 pub(crate) fn str_encode_leaf(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
     let (Object::Str(s), rest) = args.split_first()? else {
         return None;
     };
-    if rest.len() > 1 {
+    if rest.len() > 1 && !(rest.len() == 2 && is_builtin_error_handler(&rest[1])) {
         return None;
     }
     let s: &str = s;
@@ -13797,7 +13852,7 @@ pub(crate) fn bytes_decode_leaf(args: &[Object]) -> Option<Result<Object, Runtim
     let (Object::Bytes(b), rest) = args.split_first()? else {
         return None;
     };
-    if rest.len() > 1 {
+    if rest.len() > 1 && !(rest.len() == 2 && is_builtin_error_handler(&rest[1])) {
         return None;
     }
     let text = match plain_codec(rest.first())? {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 9848fb6b..a0f461af 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -18823,7 +18823,11 @@ impl Interpreter {
                 return crate::stdlib::abc_mod::abc_instance_cached(cls, &inst.cls())
                     .map(|b| Ok(Object::Bool(b)));
             }
-            return ic.is_subclass_of(cls).then_some(Ok(Object::Bool(true)));
+            if ic.is_subclass_of(cls) {
+                return Some(Ok(Object::Bool(true)));
+            }
+            // A miss is final when `__class__` can't answer otherwise.
+            return ic.class_attr_is_plain().then_some(Ok(Object::Bool(false)));
         }
         // One class, or a flat tuple of them (`isinstance(x, (str, int))`
         // asks each in turn).
@@ -38996,8 +39000,12 @@ impl Interpreter {
         }
         // Only `Instance`s can carry a custom `__class__`; for every other
         // object the real type *is* `__class__`, so skip the (observable)
-        // attribute access on the negative path.
-        if let Object::Instance(_) = obj {
+        // attribute access on the negative path. So is it for an instance
+        // whose class overrides neither `__class__` nor `__getattribute__`.
+        if let Object::Instance(inst) = obj {
+            if inst.cls().class_attr_is_plain() {
+                return Ok(Object::Bool(false));
+            }
             match self.load_attr(obj, "__class__") {
                 Ok(Object::Type(c)) => {
                     if !Rc::ptr_eq(&c, &real) && c.is_subclass_of(cls) {
@@ -52093,6 +52101,11 @@ impl Interpreter {
         kwargs: &[(String, Object)],
     ) -> Result<Object, RuntimeError> {
         if let ([arg], []) = (args, kwargs) {
+            // `type(x)`: the class itself (`b_type`), without the
+            // constructor dispatch `instantiate` runs first.
+            if Rc::ptr_eq(ty, &builtin_types().type_) {
+                return Ok(Object::Type(builtins::class_of(arg)));
+            }
             if let Some(r) = self.container_ctor1(ty, arg) {
                 return r;
             }
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 4d0f1016..b8feaeb3 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -51,10 +51,11 @@ pub enum Dunder {
     Call,
     Set,
     Delete,
+    Class,
 }
 
 impl Dunder {
-    pub const COUNT: usize = 11;
+    pub const COUNT: usize = 12;
 
     pub const fn name(self) -> &'static str {
         match self {
@@ -69,6 +70,7 @@ impl Dunder {
             Self::Call => "__call__",
             Self::Set => "__set__",
             Self::Delete => "__delete__",
+            Self::Class => "__class__",
         }
     }
 }
@@ -1881,6 +1883,16 @@ impl TypeObject {
         info
     }
 
+    /// Whether `inst.__class__` on an instance of this type is
+    /// `object`'s descriptor answering the real class: neither
+    /// `__class__` nor `__getattribute__` is overridden anywhere in the
+    /// MRO. An `isinstance` miss then needs no `__class__` read.
+    #[inline]
+    pub fn class_attr_is_plain(&self) -> bool {
+        self.dunder(Dunder::Class).object_owner()
+            && self.dunder(Dunder::GetAttribute).object_owner()
+    }
+
     pub fn lookup_with_owner(&self, name: &str) -> Option<(Object, Rc<TypeObject>)> {
         // Fast pass — see `lookup` for the gate rationale.
         if !crate::object::exotic_str_keys_possible() {
diff --git a/tests/regrtest/test_isinstance_class_attr_miss.py b/tests/regrtest/test_isinstance_class_attr_miss.py
new file mode 100644
index 00000000..60248d6f
--- /dev/null
+++ b/tests/regrtest/test_isinstance_class_attr_miss.py
@@ -0,0 +1,117 @@
+"""`isinstance` misses on instances skip the `__class__` read only when
+nothing can answer it differently.
+
+An instance whose class overrides neither `__class__` nor
+`__getattribute__` is an instance of exactly its MRO. A `__class__`
+property, a `__getattribute__` that reports another class, and
+a `__getattribute__` installed on a base after warm-up must still be consulted.
+"""
+
+import unittest
+
+
+class Plain:
+    pass
+
+
+class Liar:
+    @property
+    def __class__(self):
+        return str
+
+
+class Sneaky:
+    def __getattribute__(self, name):
+        if name == "__class__":
+            return int
+        return object.__getattribute__(self, name)
+
+
+class IsinstanceClassAttrTest(unittest.TestCase):
+    def test_plain_miss(self):
+        p = Plain()
+        for _ in range(2000):
+            self.assertFalse(isinstance(p, str))
+            self.assertFalse(isinstance(p, (int, str)))
+            self.assertTrue(isinstance(p, Plain))
+
+    def test_property_and_getattribute(self):
+        for _ in range(2000):
+            self.assertTrue(isinstance(Liar(), str))
+            self.assertTrue(isinstance(Sneaky(), int))
+            self.assertFalse(isinstance(Sneaky(), str))
+
+    def test_installed_after_warmup(self):
+        class Base:
+            pass
+
+        class Child(Base):
+            pass
+
+        c = Child()
+        for _ in range(2000):
+            self.assertFalse(isinstance(c, bytes))
+
+        def getattribute(self, name):
+            if name == "__class__":
+                return float
+            return object.__getattribute__(self, name)
+
+        Base.__getattribute__ = getattribute
+        for _ in range(2000):
+            self.assertTrue(isinstance(c, float))
+        del Base.__getattribute__
+        for _ in range(2000):
+            self.assertFalse(isinstance(c, float))
+
+    def test_class_assignment(self):
+        class A:
+            pass
+
+        class B:
+            pass
+
+        a = A()
+        for _ in range(500):
+            self.assertFalse(isinstance(a, B))
+        a.__class__ = B
+        self.assertTrue(isinstance(a, B))
+        self.assertFalse(isinstance(a, A))
+
+
+class FloatRatioAndCodecTest(unittest.TestCase):
+    def test_as_integer_ratio(self):
+        from fractions import Fraction
+        import math
+        values = [0.0, -0.0, 1.0, -1.0, 0.5, 12.3, -99.9, 1e-300, 5e-324,
+                  2.0 ** 62, 2.0 ** 63, 2.0 ** 64, 1e300, 3.141592653589793,
+                  2.0 ** -62, 2.0 ** -63, 1.5 * 2.0 ** -62, 1.0 / 3, 1e16 + 2]
+        for v in values:
+            n, d = v.as_integer_ratio()
+            self.assertIs(type(n), int)
+            self.assertIs(type(d), int)
+            self.assertGreater(d, 0)
+            self.assertEqual(math.gcd(n, d), 1 if n else d)
+            self.assertEqual(Fraction(n, d), Fraction(v))
+        self.assertEqual((-0.0).as_integer_ratio(), (0, 1))
+        self.assertEqual((0.25).as_integer_ratio(), (1, 4))
+
+    def test_encode_decode_with_errors(self):
+        for _ in range(500):
+            self.assertEqual("abc".encode("utf-8", "strict"), b"abc")
+            self.assertEqual("\xe9".encode("latin-1", "strict"), b"\xe9")
+            self.assertEqual("\xe9".encode("ascii", "replace"), b"?")
+            self.assertEqual("\xe9".encode("ascii", "ignore"), b"")
+            self.assertEqual(b"abc".decode("utf-8", "strict"), "abc")
+            self.assertEqual(b"\xff".decode("utf-8", "replace"), "�")
+            self.assertEqual(b"\xff".decode("ascii", "surrogateescape"), "\udcff")
+        with self.assertRaises(UnicodeEncodeError):
+            "\xe9".encode("ascii", "strict")
+        with self.assertRaises(UnicodeDecodeError):
+            b"\xff".decode("utf-8", "strict")
+        with self.assertRaises(LookupError):
+            "\xe9".encode("ascii", "no-such-handler")
+
+
+if __name__ == "__main__":
+    unittest.main()

From e2c184ffeedf718d0a25d2acaaa02166ba25c628 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 05:50:11 -0700
Subject: [PATCH 059/173] perf: constant-time type cache names and memoized
 class verdicts

The type cache hashes a name from its length and first and last eight
bytes instead of every byte, compares names of up to sixteen bytes in
line, and serves a hit through guard-free views of the MRO and the
owner's dict.

Per-class yes/no questions that depend only on a class's attributes
are memoized in the same cache under the class version with a family
tag in its top byte: whether getattr's leaf can call a dunder certainly
missing (copy's __deepcopy__ probe), and whether object.__reduce_ex__
reduces an instance plainly through copyreg.__newobj__. A class or base
change bumps the version, so the verdicts follow it.

object.__reduce_ex__ no longer clones the class name of an instance to
rule out builtin callables.
---
 crates/weavepy-vm/src/lib.rs              | 108 ++++++++++++---------
 crates/weavepy-vm/src/types.rs            | 110 +++++++++++++++++++++-
 tests/regrtest/test_class_verdict_memo.py |  73 ++++++++++++++
 3 files changed, 244 insertions(+), 47 deletions(-)
 create mode 100644 tests/regrtest/test_class_verdict_memo.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index a0f461af..c2ee01cb 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -58716,6 +58716,10 @@ impl Interpreter {
         &mut self,
         recv: &Object,
     ) -> Result<Option<Object>, RuntimeError> {
+        // (No class of these is subclassable: an instance is none of them.)
+        if matches!(recv, Object::Instance(_)) {
+            return Ok(None);
+        }
         let ty_name = crate::builtins::class_of(recv).name.clone();
         if !matches!(
             ty_name.as_str(),
@@ -58854,31 +58858,41 @@ impl Interpreter {
             return None;
         }
         let cls = inst.cls();
-        let slots_key = crate::object::StrKey("__slots__");
-        if cls
-            .mro
-            .borrow()
-            .iter()
-            .any(|c| c.dict.borrow().get(&slots_key).is_some())
-        {
-            return None;
-        }
-        if cls.lookup("__getnewargs_ex__").is_some() || cls.lookup("__getnewargs__").is_some() {
+        // (A pure function of the class's attributes: memoized.)
+        let plain = cls.memo_verdict(crate::types::Verdict::PlainNewobj, "", || {
+            let slots_key = crate::object::StrKey("__slots__");
+            if cls
+                .mro
+                .borrow()
+                .iter()
+                .any(|c| c.dict.borrow().get(&slots_key).is_some())
+            {
+                return false;
+            }
+            if cls.lookup("__getnewargs_ex__").is_some() || cls.lookup("__getnewargs__").is_some()
+            {
+                return false;
+            }
+            let Some(default_getstate) = builtin_types()
+                .object_
+                .dict
+                .borrow()
+                .get(&crate::object::StrKey("__getstate__"))
+                .cloned()
+            else {
+                return false;
+            };
+            // (The class lookup skips `object`'s own entry: none found is
+            // the default too.)
+            match (cls.lookup("__getstate__"), &default_getstate) {
+                (None, _) => true,
+                (Some(Object::Builtin(a)), Object::Builtin(b)) => Rc::ptr_eq(&a, b),
+                _ => false,
+            }
+        });
+        if !plain {
             return None;
         }
-        let default_getstate = builtin_types()
-            .object_
-            .dict
-            .borrow()
-            .get(&crate::object::StrKey("__getstate__"))
-            .cloned()?;
-        // (The class lookup skips `object`'s own entry: none found is the
-        // default too.)
-        match (cls.lookup("__getstate__"), &default_getstate) {
-            (None, _) => {}
-            (Some(Object::Builtin(a)), Object::Builtin(b)) if Rc::ptr_eq(&a, b) => {}
-            _ => return None,
-        }
         let newobj = self.module_attr("copyreg", "__newobj__")?;
         let dict = inst.dict_shared();
         let state = if dict.try_borrow().ok()?.is_empty() {
@@ -67114,29 +67128,39 @@ fn attr_certainly_missing(obj: &Object, name: &str) -> bool {
         return false;
     }
     let cls = inst.cls();
-    if cls.native_kind.get() != 0
-        || !cls.dunder(Dunder::GetAttribute).object_owner()
-        || cls.dunder(Dunder::GetAttr).present()
-        || cls.lookup(name).is_some()
-    {
+    if cls.native_kind.get() != 0 {
         return false;
     }
-    // A dunder may also come from a builtin base's slot (`__repr__`,
-    // `__reduce_ex__`), or be an instance-level special; one that none
-    // supplies (`__deepcopy__`, which `copy` probes) is missing.
-    if dunder {
-        let special = matches!(
-            name,
-            "__class__" | "__dict__" | "__weakref__" | "__doc__" | "__module__"
-        );
-        let builtin_base = cls
-            .mro
-            .borrow()
-            .iter()
-            .any(|t| t.flags.is_builtin && t.name != "object");
-        if special || builtin_base || builtin_slot_wrapper(&cls, name).is_some() {
+    // The class's part of the verdict depends only on its attributes
+    // (memoized until it or a base changes).
+    let class_missing = cls.memo_verdict(crate::types::Verdict::AttrMissing, name, || {
+        if !cls.dunder(Dunder::GetAttribute).object_owner()
+            || cls.dunder(Dunder::GetAttr).present()
+            || cls.lookup(name).is_some()
+        {
             return false;
         }
+        // A dunder may also come from a builtin base's slot (`__repr__`,
+        // `__reduce_ex__`), or be an instance-level special; one that none
+        // supplies (`__deepcopy__`, which `copy` probes) is missing.
+        if dunder {
+            let special = matches!(
+                name,
+                "__class__" | "__dict__" | "__weakref__" | "__doc__" | "__module__"
+            );
+            let builtin_base = cls
+                .mro
+                .borrow()
+                .iter()
+                .any(|t| t.flags.is_builtin && t.name != "object");
+            if special || builtin_base || builtin_slot_wrapper(&cls, name).is_some() {
+                return false;
+            }
+        }
+        true
+    });
+    if !class_missing {
+        return false;
     }
     match inst.dict.published() {
         Some(d) => d
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index b8feaeb3..dec85835 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -146,6 +146,18 @@ impl DunderInfo {
     }
 }
 
+/// The questions [`TypeObject::memo_verdict`] memoizes per class.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[repr(u8)]
+pub(crate) enum Verdict {
+    /// An instance attribute `name` that no class-level source supplies
+    /// (see `attr_certainly_missing`).
+    AttrMissing = 1,
+    /// The class reduces as `copyreg._reduce_newobj` would for a plain
+    /// instance (see `Interpreter::reduce_newobj_plain`); `name` unused.
+    PlainNewobj = 2,
+}
+
 pub struct AttrVersion(std::sync::atomic::AtomicU64);
 
 impl AttrVersion {
@@ -268,7 +280,6 @@ mod attr_version_tests {
 /// whose hash and length happen to collide.
 pub(crate) mod type_cache {
     use std::cell::RefCell;
-    use std::hash::Hasher;
     use std::rc::Rc;
 
     /// Entries per thread (CPython's `MCACHE_SIZE_EXP` is 12 too). The
@@ -307,11 +318,50 @@ pub(crate) mod type_cache {
         Absent,
     }
 
+    /// A slot hash for `name` from its length and at most its first and
+    /// last eight bytes: constant time for any name (entries still
+    /// compare the whole name), and attribute names differ there.
     #[inline]
     pub(crate) fn name_hash(name: &str) -> u64 {
-        let mut h = crate::fasthash::FxHasher::default();
-        h.write(name.as_bytes());
-        h.finish()
+        let b = name.as_bytes();
+        let n = b.len();
+        let (head, tail) = if n >= 8 {
+            (
+                u64::from_le_bytes(b[..8].try_into().expect("8 bytes")),
+                u64::from_le_bytes(b[n - 8..].try_into().expect("8 bytes")),
+            )
+        } else {
+            let mut w = [0u8; 8];
+            w[..n].copy_from_slice(b);
+            (u64::from_le_bytes(w), 0)
+        };
+        let x = (head ^ tail.rotate_left(29) ^ (n as u64)).wrapping_mul(0x9E37_79B9_7F4A_7C15);
+        x ^ (x >> 29)
+    }
+
+    /// `a == b` for attribute names, in line for the short ones (most
+    /// dunders and identifiers fit in sixteen bytes).
+    #[inline]
+    pub(crate) fn name_eq(a: &str, b: &str) -> bool {
+        let (a, b) = (a.as_bytes(), b.as_bytes());
+        let n = a.len();
+        if n != b.len() {
+            return false;
+        }
+        if n > 16 {
+            return a == b;
+        }
+        let w = |s: &[u8], at: usize| u64::from_le_bytes(s[at..at + 8].try_into().expect("8"));
+        if n >= 8 {
+            return w(a, 0) == w(b, 0) && w(a, n - 8) == w(b, n - 8);
+        }
+        let mut x = 0u64;
+        let mut y = 0u64;
+        for i in 0..n {
+            x |= u64::from(a[i]) << (8 * i);
+            y |= u64::from(b[i]) << (8 * i);
+        }
+        x == y
     }
 
     #[inline]
@@ -329,7 +379,7 @@ pub(crate) mod type_cache {
             let c = c.try_borrow().ok()?;
             let table = c.as_ref()?;
             let e = &table[slot(ty, ver, name_hash)];
-            if e.ver == ver && e.name.as_deref() == Some(name) {
+            if e.ver == ver && e.name.as_deref().is_some_and(|n| name_eq(n, name)) {
                 Some(if e.mro_idx == ABSENT {
                     Hit::Absent
                 } else {
@@ -1914,6 +1964,43 @@ impl TypeObject {
         None
     }
 
+    /// A per-class yes/no verdict about `name`, memoized in the type
+    /// cache until the class or a base changes: `compute` runs only on
+    /// a miss, and must be a pure function of the class's attributes
+    /// (it runs no Python code). Each [`Verdict`] family's entries live
+    /// under the class's version with the family in its top byte, which
+    /// no real version has (the dunder memo already keeps versions below
+    /// 2**56), so they never answer an attribute lookup or another
+    /// family's question.
+    #[inline]
+    pub(crate) fn memo_verdict(
+        &self,
+        family: Verdict,
+        name: &str,
+        compute: impl FnOnce() -> bool,
+    ) -> bool {
+        let ty = std::ptr::from_ref::<TypeObject>(self) as usize;
+        let ver = self.attr_version.get() | (family as u64) << 56;
+        let h = type_cache::name_hash(name);
+        match type_cache::probe(ty, ver, name, h) {
+            Some(type_cache::Hit::Absent) => true,
+            Some(type_cache::Hit::At { .. }) => false,
+            None => {
+                let verdict = compute();
+                let hit = if verdict {
+                    type_cache::Hit::Absent
+                } else {
+                    type_cache::Hit::At {
+                        mro_idx: 0,
+                        dict_idx: 0,
+                    }
+                };
+                type_cache::fill(ty, ver, name, h, hit);
+                verdict
+            }
+        }
+    }
+
     /// The plain-`str`-keys MRO walk behind [`Self::lookup`] and
     /// [`Self::lookup_with_owner`], fronted by the per-thread
     /// [`type_cache`]. Returns the value and the MRO index of its owner.
@@ -1925,6 +2012,19 @@ impl TypeObject {
         match type_cache::probe(ty, ver, name, h) {
             Some(type_cache::Hit::Absent) => return None,
             Some(type_cache::Hit::At { mro_idx, dict_idx }) => {
+                // (Guard-free views when they are free: nothing below runs
+                // code before the value is cloned.)
+                // SAFETY: both views end before anything else runs.
+                if let Some(owner) = unsafe { self.mro.peek() }.and_then(|m| m.get(mro_idx as usize))
+                {
+                    if let Some(d) = unsafe { owner.dict.peek() } {
+                        if let Some((k, v)) = d.get_index(dict_idx as usize) {
+                            if matches!(&k.0, Object::Str(s) if type_cache::name_eq(s, name)) {
+                                return Some((v.clone(), mro_idx as usize));
+                            }
+                        }
+                    }
+                }
                 let mro = self.mro.borrow();
                 if let Some(owner) = mro.get(mro_idx as usize) {
                     let d = owner.dict.borrow();
diff --git a/tests/regrtest/test_class_verdict_memo.py b/tests/regrtest/test_class_verdict_memo.py
new file mode 100644
index 00000000..fc6c51c3
--- /dev/null
+++ b/tests/regrtest/test_class_verdict_memo.py
@@ -0,0 +1,73 @@
+"""Per-class verdicts memoized in the type cache follow class changes.
+
+`getattr(obj, "__deepcopy__", None)` decides a missing dunder from the
+class once per class version, and `object.__reduce_ex__` decides the
+plain `copyreg.__newobj__` reduction the same way. Adding the attribute
+to the class or a base, a `__getattr__`, a `__getstate__`, or
+`__getnewargs__` after warm-up must change the answer, and an instance
+attribute must still win per instance.
+"""
+
+import copy
+import copyreg
+import unittest
+
+
+class VerdictMemoTest(unittest.TestCase):
+    def test_missing_dunder_follows_class_changes(self):
+        class Base:
+            pass
+
+        class Leaf(Base):
+            pass
+
+        obj = Leaf()
+        for _ in range(500):
+            self.assertIsNone(getattr(obj, "__deepcopy__", None))
+            self.assertFalse(hasattr(obj, "__deepcopy__"))
+        Base.__deepcopy__ = lambda self, memo: "base"
+        for _ in range(500):
+            self.assertEqual(obj.__deepcopy__({}), "base")
+            self.assertIsNotNone(getattr(obj, "__deepcopy__", None))
+        del Base.__deepcopy__
+        for _ in range(500):
+            self.assertIsNone(getattr(obj, "__deepcopy__", None))
+        obj.__dict__["__deepcopy__"] = "own"
+        self.assertEqual(getattr(obj, "__deepcopy__", None), "own")
+        self.assertIsNone(getattr(Leaf(), "__deepcopy__", None))
+        Base.__getattr__ = lambda self, name: "dynamic"
+        self.assertEqual(getattr(Leaf(), "__deepcopy__", None), "dynamic")
+
+    def test_plain_reduction_follows_class_changes(self):
+        class Base:
+            pass
+
+        class Point(Base):
+            def __init__(self, x):
+                self.x = x
+
+        p = Point(3)
+        for _ in range(500):
+            rv = p.__reduce_ex__(4)
+            self.assertIs(rv[0], copyreg.__newobj__)
+            self.assertEqual(rv[1], (Point,))
+            self.assertEqual(rv[2], {"x": 3})
+            self.assertEqual(copy.deepcopy(p).x, 3)
+        Base.__getstate__ = lambda self: {"x": 42}
+        for _ in range(500):
+            self.assertEqual(p.__reduce_ex__(4)[2], {"x": 42})
+            self.assertEqual(copy.deepcopy(p).x, 42)
+        del Base.__getstate__
+        Base.__getnewargs__ = lambda self: (7,)
+        rv = p.__reduce_ex__(4)
+        self.assertEqual(rv[1], (Point, 7))
+        del Base.__getnewargs__
+        self.assertEqual(p.__reduce_ex__(4)[1], (Point,))
+
+    def test_builtin_function_reduction(self):
+        self.assertEqual(len.__reduce_ex__(4), "len")
+        self.assertEqual(copy.deepcopy(len), len)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 74651817cf5e2f2ccae9f4e55caed6f5b2a2776c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 06:50:43 -0700
Subject: [PATCH 060/173] perf: leaf builtin dunders, positional startswith,
 shared chars, gcd

getattr's leaf answers a dunder a plain instance inherits as a builtin
method (copy's getattr(x, '__reduce_ex__', None)) as the full lookup
does: an instance-dict entry, else the method bound to the instance.
Weak proxies, super proxies and classes with __getattr__ or a custom
__getattribute__ keep the full path.

Per-class verdict entries hash to their own type cache slots: they
shared the attribute lookup's slot for the same name and evicted it on
every call.

str.startswith/endswith with a start position skip the per-call ASCII
scan of the prefix when the receiver is known to be ASCII (its
code-point count is memoized), so tomllib's tokenizer no longer pays
for the position on every probe.

Object::from_char hands out the shared ASCII string by index.

math.gcd runs a binary GCD over machine words, and for big operands
strips the common power of two first (a float ratio's denominator is
one), finishing in 128-bit arithmetic or after one remainder when an
operand is small.
---
 crates/weavepy-vm/src/builtins.rs        | 27 +++++--
 crates/weavepy-vm/src/lib.rs             | 46 ++++++++++--
 crates/weavepy-vm/src/object.rs          |  6 +-
 crates/weavepy-vm/src/stdlib/math.rs     | 79 +++++++++++++++++++-
 crates/weavepy-vm/src/types.rs           |  7 +-
 tests/regrtest/test_library_fastpaths.py | 91 ++++++++++++++++++++++++
 6 files changed, 243 insertions(+), 13 deletions(-)
 create mode 100644 tests/regrtest/test_library_fastpaths.py

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 6fbea137..91cd16fa 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -6064,7 +6064,10 @@ fn float_as_integer_ratio(args: &[Object]) -> Result<Object, RuntimeError> {
         };
         if let Some((Some(num), Some(den))) = small {
             let num = if sign < 0 { -num } else { num };
-            return Ok(Object::new_tuple_array([Object::Int(num), Object::Int(den)]));
+            return Ok(Object::new_tuple_array([
+                Object::Int(num),
+                Object::Int(den),
+            ]));
         }
     }
     let (mantissa, exponent): (BigInt, i32) = if exp_field == 0 {
@@ -11435,7 +11438,7 @@ fn str_startswith(args: &[Object]) -> Result<Object, RuntimeError> {
         Some(obj) => obj,
         None => return Err(type_error("startswith() takes at least 1 argument")),
     };
-    let slice = str_apply_start_end(s.as_ref(), args.get(2), args.get(3))?;
+    let slice = str_apply_start_end(s.as_ref(), args.get(2), args.get(3), ascii_receiver(args))?;
     match slice {
         Some(slice) => Ok(Object::Bool(str_match_prefix_suffix(slice, target, true)?)),
         None => Ok(Object::Bool(false)),
@@ -11449,13 +11452,28 @@ fn str_endswith(args: &[Object]) -> Result<Object, RuntimeError> {
         Some(obj) => obj,
         None => return Err(type_error("endswith() takes at least 1 argument")),
     };
-    let slice = str_apply_start_end(s.as_ref(), args.get(2), args.get(3))?;
+    let slice = str_apply_start_end(s.as_ref(), args.get(2), args.get(3), ascii_receiver(args))?;
     match slice {
         Some(slice) => Ok(Object::Bool(str_match_prefix_suffix(slice, target, false)?)),
         None => Ok(Object::Bool(false)),
     }
 }
 
+/// Whether a `str` method's receiver is known to be pure ASCII, so a
+/// character index is a byte index. A long receiver's code points are
+/// counted once (the count is memoized on the string); a scan per call
+/// would make `s.startswith(x, pos)` linear in `pos`, as tomllib's
+/// tokenizer calls it at every position.
+fn ascii_receiver(args: &[Object]) -> bool {
+    match args.first() {
+        Some(Object::Str(t)) => {
+            crate::shared_value::SharedStr::known_char_count(t) == Some(t.len())
+                || (t.len() > 64 && crate::shared_value::SharedStr::is_ascii(t))
+        }
+        _ => false,
+    }
+}
+
 /// Resolve start/end for `startswith`/`endswith` like CPython's
 /// `tailmatch` + `ADJUST_INDICES`: `end` clamps to the length, but
 /// `start` is only floored at 0 — a start beyond the (adjusted) end
@@ -11465,6 +11483,7 @@ fn str_apply_start_end<'a>(
     s: &'a str,
     start: Option<&Object>,
     end: Option<&Object>,
+    ascii: bool,
 ) -> Result<Option<&'a str>, RuntimeError> {
     // The common unbounded check only needs to inspect the needle. In
     // particular, json.loads checks for a BOM this way on the whole input.
@@ -11504,7 +11523,7 @@ fn str_apply_start_end<'a>(
         }
         let index = usize::try_from(index).ok()?;
         let prefix = text.as_bytes().get(..index)?;
-        if prefix.is_ascii() {
+        if ascii || prefix.is_ascii() {
             Some(index)
         } else {
             text.char_indices()
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index c2ee01cb..0327fa50 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -26189,8 +26189,10 @@ impl Interpreter {
         // A dunder nothing supplies (`copy`'s `__deepcopy__` probe): the
         // default.
         if name.starts_with("__") {
-            return (args.len() == 3 && attr_certainly_missing(&args[0], name))
-                .then(|| Ok(args[2].clone()));
+            if args.len() == 3 && attr_certainly_missing(&args[0], name) {
+                return Some(Ok(args[2].clone()));
+            }
+            return Self::leaf_builtin_dunder_attr(inst, &args[0], name).map(Ok);
         }
         let cls = inst.cls_raw();
         if cls.native_kind.get() != 0
@@ -26232,6 +26234,43 @@ impl Interpreter {
         }))
     }
 
+    /// A dunder a plain instance gets from a builtin method on its class
+    /// (`obj.__reduce_ex__`, which `copy` fetches): what
+    /// `load_attr_instance_default` resolves for it (an instance-dict
+    /// entry, else the method bound to `obj`), or `None` for any shape
+    /// that path treats specially.
+    fn leaf_builtin_dunder_attr(inst: &PyInstance, recv: &Object, name: &str) -> Option<Object> {
+        let cls = inst.cls_raw();
+        if cls.native_kind.get() != 0
+            || inst.native.get().is_some()
+            || inst.c_body.get() != 0
+            || !Self::default_getattribute(cls)
+            || cls.dunder(crate::types::Dunder::GetAttr).present()
+            || cls.is_super_proxy_type()
+        {
+            return None;
+        }
+        let Some(Object::Builtin(b)) = cls.lookup(name) else {
+            return None;
+        };
+        // (A weak proxy forwards these names to its referent.)
+        if matches!(
+            name,
+            "__reduce_ex__" | "__reduce__" | "__copy__" | "__deepcopy__" | "__getstate__"
+        ) && crate::stdlib::weakref_real::proxy_referent(recv).is_some()
+        {
+            return None;
+        }
+        if let Some(v) = inst.attr_get_str(name) {
+            return Some(v);
+        }
+        Some(if b.binds_instance {
+            Object::BoundMethod(Rc::new(BoundMethod::new(recv.clone(), Object::Builtin(b))))
+        } else {
+            Object::Builtin(b)
+        })
+    }
+
     /// `hasattr(obj, name)` decided as [`Self::leaf_getattr`] decides
     /// `getattr(obj, name, default)`: `True` for a value it finds, `False`
     /// for a name it proves missing. `None` for anything the full call
@@ -58869,8 +58908,7 @@ impl Interpreter {
             {
                 return false;
             }
-            if cls.lookup("__getnewargs_ex__").is_some() || cls.lookup("__getnewargs__").is_some()
-            {
+            if cls.lookup("__getnewargs_ex__").is_some() || cls.lookup("__getnewargs__").is_some() {
                 return false;
             }
             let Some(default_getstate) = builtin_types()
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 58600920..c3295178 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -14003,7 +14003,11 @@ impl Object {
     /// allocates nothing for them.
     #[inline]
     pub fn from_char(ch: char) -> Self {
-        // (Latin-1 characters come out of `SharedStr::small`'s table.)
+        // (Latin-1 characters come out of `SharedStr::small`'s table; an
+        // ASCII one is its table entry by index.)
+        if ch.is_ascii() {
+            return Object::Str(SharedStr::ascii_char(ch as u8));
+        }
         Object::Str(SharedStr::from(&*ch.encode_utf8(&mut [0; 4])))
     }
 
diff --git a/crates/weavepy-vm/src/stdlib/math.rs b/crates/weavepy-vm/src/stdlib/math.rs
index 851c26ec..43cc7d85 100644
--- a/crates/weavepy-vm/src/stdlib/math.rs
+++ b/crates/weavepy-vm/src/stdlib/math.rs
@@ -978,15 +978,90 @@ fn libm_fmod(x: f64, y: f64) -> f64 {
 }
 
 fn math_gcd(args: &[Object]) -> Result<Object, RuntimeError> {
-    use num_integer::Integer;
+    // Machine-word arguments, the common call: a binary GCD of their
+    // magnitudes (`gcd(i64::MIN, 0)` is 2**63, past `i64`).
+    let small = |o: &Object| match o {
+        Object::Int(i) => Some(i.unsigned_abs()),
+        Object::Bool(b) => Some(u64::from(*b)),
+        _ => None,
+    };
+    if let Some(words) = args.iter().map(small).collect::<Option<Vec<u64>>>() {
+        let g = words
+            .into_iter()
+            .fold(0u64, |a, b| gcd_u128(u128::from(a), u128::from(b)) as u64);
+        return Ok(i64::try_from(g).map_or_else(
+            |_| Object::int_from_bigint(num_bigint::BigInt::from(g)),
+            Object::Int,
+        ));
+    }
     let mut acc = num_bigint::BigInt::from(0);
     for a in args {
         let v = index_bigint(a)?;
-        acc = acc.gcd(&v); // num_integer::gcd is always non-negative
+        acc = gcd_bigint(&acc, &v);
     }
     Ok(Object::int_from_bigint(acc))
 }
 
+/// Binary GCD of two machine words.
+fn gcd_u128(mut a: u128, mut b: u128) -> u128 {
+    if a == 0 {
+        return b;
+    }
+    if b == 0 {
+        return a;
+    }
+    let shift = (a | b).trailing_zeros();
+    a >>= a.trailing_zeros();
+    loop {
+        b >>= b.trailing_zeros();
+        if a > b {
+            std::mem::swap(&mut a, &mut b);
+        }
+        b -= a;
+        if b == 0 {
+            return a << shift;
+        }
+    }
+}
+
+/// The non-negative GCD of two integers (`num_integer`'s answer), with
+/// the shapes `fractions` and `statistics` produce settled cheaply: a
+/// power-of-two factor (every float's ratio has a power-of-two
+/// denominator), magnitudes that fit 128 bits, and one small operand
+/// (a single remainder reduces the other to a word).
+pub(crate) fn gcd_bigint(a: &num_bigint::BigInt, b: &num_bigint::BigInt) -> num_bigint::BigInt {
+    use num_bigint::{BigInt, BigUint};
+    use num_integer::Integer;
+    use num_traits::{ToPrimitive, Zero};
+    let (a, b) = (a.magnitude(), b.magnitude());
+    if a.is_zero() {
+        return BigInt::from(b.clone());
+    }
+    if b.is_zero() {
+        return BigInt::from(a.clone());
+    }
+    let (za, zb) = (
+        a.trailing_zeros().unwrap_or(0),
+        b.trailing_zeros().unwrap_or(0),
+    );
+    let shift = za.min(zb);
+    let (a, b) = (a >> za, b >> zb);
+    let odd = if a == BigUint::from(1u8) || b == BigUint::from(1u8) {
+        BigUint::from(1u8)
+    } else if let (Some(x), Some(y)) = (a.to_u128(), b.to_u128()) {
+        BigUint::from(gcd_u128(x, y))
+    } else if let Some(y) = b.to_u64() {
+        let r = (&a % y).to_u64().expect("a remainder below a word");
+        BigUint::from(gcd_u128(u128::from(r), u128::from(y)))
+    } else if let Some(x) = a.to_u64() {
+        let r = (&b % x).to_u64().expect("a remainder below a word");
+        BigUint::from(gcd_u128(u128::from(r), u128::from(x)))
+    } else {
+        a.gcd(&b)
+    };
+    BigInt::from(odd << shift)
+}
+
 fn math_lcm(args: &[Object]) -> Result<Object, RuntimeError> {
     use num_integer::Integer;
     use num_traits::{Signed, Zero};
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index dec85835..ee22275a 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -1981,7 +1981,9 @@ impl TypeObject {
     ) -> bool {
         let ty = std::ptr::from_ref::<TypeObject>(self) as usize;
         let ver = self.attr_version.get() | (family as u64) << 56;
-        let h = type_cache::name_hash(name);
+        // (The slot ignores the version's top byte: a family moves its
+        // entries off the ones the attribute lookups use.)
+        let h = type_cache::name_hash(name) ^ (family as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
         match type_cache::probe(ty, ver, name, h) {
             Some(type_cache::Hit::Absent) => true,
             Some(type_cache::Hit::At { .. }) => false,
@@ -2015,7 +2017,8 @@ impl TypeObject {
                 // (Guard-free views when they are free: nothing below runs
                 // code before the value is cloned.)
                 // SAFETY: both views end before anything else runs.
-                if let Some(owner) = unsafe { self.mro.peek() }.and_then(|m| m.get(mro_idx as usize))
+                if let Some(owner) =
+                    unsafe { self.mro.peek() }.and_then(|m| m.get(mro_idx as usize))
                 {
                     if let Some(d) = unsafe { owner.dict.peek() } {
                         if let Some((k, v)) = d.get_index(dict_idx as usize) {
diff --git a/tests/regrtest/test_library_fastpaths.py b/tests/regrtest/test_library_fastpaths.py
new file mode 100644
index 00000000..40501905
--- /dev/null
+++ b/tests/regrtest/test_library_fastpaths.py
@@ -0,0 +1,91 @@
+"""Native shortcuts behind library-heavy code keep their exact answers.
+
+- `getattr(obj, dunder)` for a builtin method a plain instance inherits
+  (`copy` fetches `__reduce_ex__` this way) binds as the full lookup does,
+  defers to an instance-dict entry, and leaves weak proxies alone.
+- `str.startswith`/`endswith` with a start position on long strings, ASCII
+  or not, count characters, not bytes.
+- One-character strings from indexing are the shared Latin-1 objects.
+- `math.gcd` over machine words, 128-bit values, power-of-two factors,
+  and mixed big and small operands.
+"""
+
+import math
+import random
+import unittest
+import weakref
+
+
+class Plain:
+    pass
+
+
+class LibraryFastpathTest(unittest.TestCase):
+    def test_getattr_builtin_dunder(self):
+        p = Plain()
+        for _ in range(1000):
+            m = getattr(p, "__reduce_ex__", None)
+            self.assertIs(m.__self__, p)
+            self.assertEqual(m, object.__getattribute__(p, "__reduce_ex__"))
+            self.assertTrue(hasattr(p, "__reduce_ex__"))
+            self.assertEqual(getattr(p, "__format__")(""), str(p))
+        p.__dict__["__reduce_ex__"] = "mine"
+        self.assertEqual(getattr(p, "__reduce_ex__", None), "mine")
+        target = Plain()
+        proxy = weakref.proxy(target)
+        self.assertIs(getattr(proxy, "__reduce_ex__").__self__, target)
+
+    def test_startswith_positions(self):
+        ascii_text = "key = value\n" * 20
+        wide_text = "été = ☃\n" * 20
+        for text in (ascii_text, wide_text):
+            n = len(text)
+            for pos in range(0, n + 1, 7):
+                for needle in ("key", "ét", "=", "\n", ""):
+                    self.assertEqual(
+                        text.startswith(needle, pos), text[pos:].startswith(needle)
+                    )
+                    self.assertEqual(
+                        text.endswith(needle, 0, pos), text[:pos].endswith(needle)
+                    )
+            self.assertFalse(text.startswith("", n + 1))
+            self.assertTrue(text.startswith(("zz", text[5]), 5))
+        self.assertTrue(wide_text.startswith("☃", 6))
+
+    def test_index_chars_are_shared(self):
+        s = "abcé" * 50
+        for i in range(len(s)):
+            c = s[i]
+            self.assertEqual(c, chr(ord(c)))
+        self.assertIs(s[0], "a")
+        self.assertIs(s[3], "é")
+        self.assertIs("".join(["x", "y"])[1], "y")
+
+    def test_gcd(self):
+        r = random.Random(7)
+        cases = [(0, 0), (0, 5), (-6, 0), (-2**63, 0), (-2**63, -2**63),
+                 (True, 4), (False, False), (2**200, 6 * 2**150), (3**80, 3**40 * 7),
+                 (2**64 + 1, 2**64 - 1), (25, 2**300 * 5)]
+        for _ in range(500):
+            a = r.getrandbits(r.choice([5, 60, 64, 100, 128, 200])) << r.randrange(70)
+            b = r.getrandbits(r.choice([3, 40, 64, 130, 300])) << r.randrange(70)
+            cases.append((a * r.choice([1, -1]), b))
+        for a, b in cases:
+            g = math.gcd(a, b)
+            self.assertIs(type(g), int)
+            self.assertGreaterEqual(g, 0)
+            if g:
+                self.assertEqual(a % g, 0)
+                self.assertEqual(b % g, 0)
+                self.assertEqual(math.gcd(a // g, b // g), 1)
+            else:
+                self.assertEqual((a, b), (0, 0))
+        self.assertEqual(math.gcd(12, 18, 27), 3)
+        self.assertEqual(math.gcd(2**70 * 3, 2**65 * 9, 2**80 * 15), 2**65 * 3)
+        self.assertEqual(math.gcd(), 0)
+        with self.assertRaises(TypeError):
+            math.gcd(1.0, 2)
+
+
+if __name__ == "__main__":
+    unittest.main()

From deee93659f27801f325812a945ed00679682e1bf Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 07:42:17 -0700
Subject: [PATCH 061/173] perf: serve struct.Struct.pack and unpack natively

CPython's Struct is a C type; WeavePy's is a Python class whose pack
and unpack wrapped the native core in a Python call with a star-args
tuple, a try block and a format lookup by text. _struct now installs
native pack and unpack methods on the exact class: they read the
instance's format string, find its compiled format by the string's
identity (a small per-thread cache), and pack plain values or unpack a
bytes buffer in place. Every other shape (subclasses, values needing
__index__ or __float__, other buffers, an uninitialized Struct, and
every error) calls the Python method they replaced, so behavior is
unchanged. The fast halves are leaf builtins, so the dispatch loop runs
them in line.

bytes_codecs drops from 1.66x to 1.27x CPython's instructions.
---
 crates/weavepy-vm/src/stdlib/python/struct.py |   7 +
 crates/weavepy-vm/src/stdlib/struct_mod.rs    | 159 ++++++++++++++++++
 tests/regrtest/test_struct_native_methods.py  |  59 +++++++
 3 files changed, 225 insertions(+)
 create mode 100644 tests/regrtest/test_struct_native_methods.py

diff --git a/crates/weavepy-vm/src/stdlib/python/struct.py b/crates/weavepy-vm/src/stdlib/python/struct.py
index 2454cb1e..6897c12a 100644
--- a/crates/weavepy-vm/src/stdlib/python/struct.py
+++ b/crates/weavepy-vm/src/stdlib/python/struct.py
@@ -339,6 +339,13 @@ def __sizeof__(self):
         return object.__sizeof__(self)
 
 
+# CPython's `Struct` is a C type: serve the exact class's `pack` and
+# `unpack` natively (the methods above stay the fallback for every other
+# shape, errors included).
+if hasattr(_impl, "_install_struct"):
+    _impl._install_struct(Struct)
+
+
 def _new_struct(fmt):
     return Struct(fmt)
 
diff --git a/crates/weavepy-vm/src/stdlib/struct_mod.rs b/crates/weavepy-vm/src/stdlib/struct_mod.rs
index 2a29f96b..ddc93741 100644
--- a/crates/weavepy-vm/src/stdlib/struct_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/struct_mod.rs
@@ -915,6 +915,16 @@ pub fn build(_cache: &ModuleCache) -> Rc<PyModule> {
         register(&mut d, "pack_into", b_pack_into);
         register(&mut d, "unpack_from", b_unpack_from);
         register(&mut d, "iter_unpack", b_iter_unpack);
+        let install = Rc::new(BuiltinFn {
+            name: "_install_struct",
+            binds_instance: false,
+            call: Box::new(install_struct),
+            call_kw: None,
+        });
+        d.insert(
+            DictKey(Object::from_static("_install_struct")),
+            Object::Builtin(install),
+        );
     }
     Rc::new(PyModule {
         name: "_struct".to_owned(),
@@ -970,6 +980,155 @@ fn compiled(fmt: &str) -> Result<std::rc::Rc<CompiledFormat>, RuntimeError> {
 /// `_clearcache()`: flush the compiled formats.
 fn b_clearcache(_args: &[Object]) -> Result<Object, RuntimeError> {
     FORMATS.with(|c| c.borrow_mut().clear());
+    BY_STRING.with(|c| c.borrow_mut().clear());
+    Ok(Object::None)
+}
+
+/// The address of the exact `struct.Struct` class [`install_struct`]
+/// last served (its native methods keep it alive).
+static STRUCT_CLASS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
+
+thread_local! {
+    /// Compiled formats by the identity of a `Struct`'s format string
+    /// (each held here, so an address names one string), most recent
+    /// first: a `Struct` hands its methods the same string every call.
+    static BY_STRING: std::cell::RefCell<Vec<(crate::shared_value::SharedStr, std::rc::Rc<CompiledFormat>)>> =
+        const { std::cell::RefCell::new(Vec::new()) };
+}
+
+/// How many format strings [`BY_STRING`] remembers.
+const BY_STRING_MAX: usize = 8;
+
+/// The compiled format of a `Struct`'s format string `fmt`.
+fn compiled_for(fmt: &crate::shared_value::SharedStr) -> Option<std::rc::Rc<CompiledFormat>> {
+    let addr = crate::shared_value::SharedStr::addr(fmt);
+    let hit = BY_STRING.with(|c| {
+        c.try_borrow().ok().and_then(|c| {
+            c.iter()
+                .find(|(s, _)| crate::shared_value::SharedStr::addr(s) == addr)
+                .map(|(_, cf)| cf.clone())
+        })
+    });
+    if hit.is_some() {
+        return hit;
+    }
+    let cf = compiled(fmt).ok()?;
+    BY_STRING.with(|c| {
+        if let Ok(mut c) = c.try_borrow_mut() {
+            c.truncate(BY_STRING_MAX - 1);
+            c.insert(0, (fmt.clone(), cf.clone()));
+        }
+    });
+    Some(cf)
+}
+
+/// The compiled format of `recv` when it is an exact, initialized
+/// `struct.Struct` whose `_fmt` is a plain `str` in its own dictionary
+/// (anything else is the Python method's to serve).
+fn struct_format(recv: &Object) -> Option<std::rc::Rc<CompiledFormat>> {
+    let Object::Instance(inst) = recv else {
+        return None;
+    };
+    let class = std::ptr::from_ref(inst.cls_raw()) as usize;
+    if class != STRUCT_CLASS.load(std::sync::atomic::Ordering::Relaxed) {
+        return None;
+    }
+    match inst.attr_get_str("_fmt")? {
+        Object::Str(fmt) => compiled_for(&fmt),
+        _ => None,
+    }
+}
+
+/// `Struct.pack(self, *values)` over plain values (what `_pack_plain`
+/// packs), or `None` for the Python method, which also owns every error.
+fn struct_pack_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
+    let (recv, values) = args.split_first()?;
+    let plain = values.iter().all(|v| {
+        matches!(
+            v,
+            Object::Int(_)
+                | Object::Long(_)
+                | Object::Bool(_)
+                | Object::Float(_)
+                | Object::Complex(_)
+                | Object::Str(_)
+                | Object::Bytes(_)
+                | Object::ByteArray(_)
+        )
+    });
+    if !plain {
+        return None;
+    }
+    let cf = struct_format(recv)?;
+    let bytes = cf.pack(values).ok()?;
+    Some(Ok(Object::new_bytes(bytes)))
+}
+
+/// `Struct.unpack(self, buffer)` of a `bytes` buffer, or `None` for the
+/// Python method, which also owns every error.
+fn struct_unpack_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
+    let [recv, Object::Bytes(buf)] = args else {
+        return None;
+    };
+    let cf = struct_format(recv)?;
+    let vals = cf.unpack(buf).ok()?;
+    Some(Ok(Object::new_tuple(vals)))
+}
+
+/// `_install_struct(Struct)`: serve the exact class's `pack` and
+/// `unpack` natively (CPython's `Struct` is a C type), each falling back
+/// to the Python method it replaces for every other shape.
+fn install_struct(args: &[Object]) -> Result<Object, RuntimeError> {
+    let [Object::Type(cls)] = args else {
+        return Err(type_error("_install_struct() expects the Struct class"));
+    };
+    let own = |name: &str| cls.dict.borrow().get(&crate::object::StrKey(name)).cloned();
+    let (Some(pack), Some(unpack)) = (own("pack"), own("unpack")) else {
+        return Err(type_error("_install_struct(): Struct lacks pack/unpack"));
+    };
+    type Fast = fn(&[Object]) -> Option<Result<Object, RuntimeError>>;
+    for (name, fast, orig) in [
+        ("pack", struct_pack_fast as Fast, pack),
+        ("unpack", struct_unpack_fast as Fast, unpack),
+    ] {
+        // (The natives keep the class alive: its address stays its own.)
+        let class = cls.clone();
+        let fallback = move |a: &[Object], kw: &[(String, Object)]| {
+            let _ = &class;
+            let f = orig.clone();
+            let ptr = crate::vm_singletons::current_interpreter_ptr()
+                .ok_or_else(|| type_error("struct: no active interpreter"))?;
+            // SAFETY: published by the enclosing VM frame on this thread.
+            unsafe { &mut *ptr }.call_object(f, a, kw)
+        };
+        let fallback = Rc::new(fallback);
+        let fallback_kw = fallback.clone();
+        let b = Rc::new(BuiltinFn {
+            name,
+            binds_instance: true,
+            call: Box::new(move |a: &[Object]| match fast(a) {
+                Some(r) => r,
+                None => fallback(a, &[]),
+            }),
+            call_kw: Some(Box::new(move |a: &[Object], kw: &[(String, Object)]| {
+                if kw.is_empty() {
+                    if let Some(r) = fast(a) {
+                        return r;
+                    }
+                }
+                fallback_kw(a, kw)
+            })),
+        });
+        crate::leaf_builtins::register_fast(&b, fast);
+        cls.dict
+            .borrow_mut()
+            .insert(DictKey(Object::from_static(name)), Object::Builtin(b));
+    }
+    cls.bump_attr_version();
+    STRUCT_CLASS.store(
+        Rc::as_ptr(cls) as usize,
+        std::sync::atomic::Ordering::Relaxed,
+    );
     Ok(Object::None)
 }
 
diff --git a/tests/regrtest/test_struct_native_methods.py b/tests/regrtest/test_struct_native_methods.py
new file mode 100644
index 00000000..50b78bd6
--- /dev/null
+++ b/tests/regrtest/test_struct_native_methods.py
@@ -0,0 +1,59 @@
+"""`struct.Struct.pack`/`unpack` served natively keep the class's rules.
+
+The exact class's methods pack plain values and unpack `bytes` without
+running the Python methods, which remain the fallback: subclasses,
+`__index__` values, non-`bytes` buffers, an uninitialized or
+re-initialized `Struct`, and every error must answer as they do.
+"""
+
+import struct
+import unittest
+
+
+class StructNativeTest(unittest.TestCase):
+    def test_round_trip(self):
+        s = struct.Struct("<IhdB3s")
+        for i in range(2000):
+            packed = s.pack(i, -i & 0x7FFF, i * 0.5, i & 255, b"abc")
+            self.assertEqual(packed, struct.pack("<IhdB3s", i, -i & 0x7FFF, i * 0.5, i & 255, b"abc"))
+            self.assertEqual(s.unpack(packed), (i, -i & 0x7FFF, i * 0.5, i & 255, b"abc"))
+        self.assertEqual(s.unpack(bytearray(packed)), s.unpack(packed))
+        self.assertEqual(s.unpack(memoryview(packed)), s.unpack(packed))
+
+    def test_errors(self):
+        s = struct.Struct("<hB")
+        with self.assertRaises(struct.error):
+            s.pack(1)
+        with self.assertRaises(struct.error):
+            s.pack(1, 256)
+        with self.assertRaises(struct.error):
+            s.pack("x", 1)
+        with self.assertRaises(struct.error):
+            s.unpack(b"\x00")
+        with self.assertRaises(TypeError):
+            s.unpack("abc")
+
+    def test_fallback_shapes(self):
+        class Index:
+            def __index__(self):
+                return 7
+
+        class Sub(struct.Struct):
+            pass
+
+        self.assertEqual(struct.Struct("<h").pack(Index()), b"\x07\x00")
+        self.assertEqual(Sub("<h").pack(5), b"\x05\x00")
+        self.assertEqual(Sub("<h").unpack(b"\x05\x00"), (5,))
+        blank = struct.Struct.__new__(struct.Struct)
+        with self.assertRaises(RuntimeError):
+            blank.pack(1)
+        s = struct.Struct("<h")
+        self.assertEqual(s.pack(1), b"\x01\x00")
+        s.__init__(">h")
+        self.assertEqual(s.pack(1), b"\x00\x01")
+        self.assertEqual(s.unpack(b"\x00\x01"), (1,))
+        self.assertEqual(s.format, ">h")
+
+
+if __name__ == "__main__":
+    unittest.main()

From cfaff96b32b1244b4be099bd410440a032e39403 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 08:29:14 -0700
Subject: [PATCH 062/173] perf: one-character regex asserts in place, ASCII
 translate memo

The regex matcher entered itself for every lookaround, including those
whose body is one character test (a literal or a set, possibly
case-folded) or a position, or an alternation of them. Compiling a
pattern now records each such body as a short list of tests, which the
ASSERT and ASSERT_NOT arms evaluate in place (the bodies set no marks).
textwrap's word splitter runs two of these at every character.

str.translate looks each ASCII character's entry up once per call, as
CPython's unicode_fast_translate caches its ASCII table, and sizes its
output up front.

Loading a name off a builtin class remembers, per class version, that
its own dict holds no class method of that name, sparing the probe's
hashing on reads like object.__new__.
---
 crates/weavepy-vm/src/builtins.rs          |  15 ++-
 crates/weavepy-vm/src/lib.rs               |  12 +-
 crates/weavepy-vm/src/stdlib/sre_mod.rs    | 142 ++++++++++++++++++++-
 crates/weavepy-vm/src/types.rs             |   3 +
 tests/regrtest/test_regex_quick_asserts.py |  65 ++++++++++
 5 files changed, 225 insertions(+), 12 deletions(-)
 create mode 100644 tests/regrtest/test_regex_quick_asserts.py

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 91cd16fa..b15b93b5 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -12778,9 +12778,12 @@ fn str_translate(args: &[Object]) -> Result<Object, RuntimeError> {
             ))
         }
     };
-    let mut out = String::new();
+    let mut out = String::with_capacity(s.len());
     let receiver_bridged = matches!(args.first(), Some(Object::WStr(_)));
     let mut saw_surrogate = receiver_bridged;
+    // Each ASCII character's entry, looked up once per call (CPython's
+    // `unicode_fast_translate` caches its ASCII table the same way).
+    let mut ascii_memo: [Option<Option<Object>>; 128] = std::array::from_fn(|_| None);
     // Push a translation target code point, bridging a surrogate so it
     // round-trips through `str_result`.
     let push_cp = |out: &mut String, cp: u32, saw: &mut bool| {
@@ -12805,13 +12808,17 @@ fn str_translate(args: &[Object]) -> Result<Object, RuntimeError> {
         } else {
             cp
         };
-        let entry = match &table {
+        let lookup = |cp: u32| match &table {
             Table::Dict(Object::Dict(d)) => d
                 .borrow()
-                .get(&DictKey(Object::Int(i64::from(real_cp))))
+                .get(&DictKey(Object::Int(i64::from(cp))))
                 .cloned(),
             Table::Dict(_) => None,
-            Table::Seq(v) => v.get(real_cp as usize).cloned(),
+            Table::Seq(v) => v.get(cp as usize).cloned(),
+        };
+        let entry = match ascii_memo.get_mut(real_cp as usize) {
+            Some(slot) => slot.get_or_insert_with(|| lookup(real_cp)).clone(),
+            None => lookup(real_cp),
         };
         match entry {
             Some(Object::None) => {}
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 0327fa50..693a7a80 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -27276,8 +27276,16 @@ impl Interpreter {
         }
         // A builtin class's native class method (`int.from_bytes`), bound
         // to the class as `classmethod.__get__` binds it.
-        if cls.flags.is_builtin {
-            let name = code.names.get(name_idx as usize)?;
+        // (Most names are none: the class remembers which, sparing the
+        // probe's hashing.)
+        let name = code.names.get(name_idx as usize)?;
+        if cls.flags.is_builtin
+            && !cls.memo_verdict(crate::types::Verdict::NoNativeClassMethod, name, || {
+                cls.dict.try_borrow().is_ok_and(|d| {
+                    !matches!(d.get(&crate::object::StrKey(name)), Some(Object::ClassMethod(_)))
+                })
+            })
+        {
             let d = cls.dict.try_borrow().ok()?;
             if let Some(Object::ClassMethod(cm)) = d.get(&crate::object::StrKey(name)) {
                 let f = cm.func();
diff --git a/crates/weavepy-vm/src/stdlib/sre_mod.rs b/crates/weavepy-vm/src/stdlib/sre_mod.rs
index b7d2d7c9..de632e05 100644
--- a/crates/weavepy-vm/src/stdlib/sre_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/sre_mod.rs
@@ -146,6 +146,14 @@ pub(crate) struct CompiledCode {
     /// table of the character set starting there ([`NO_TABLE`] for none).
     table_of: Vec<u32>,
     tables: Vec<[u64; 4]>,
+    /// Per code position, the index in `quick_ranges` of the tests of the
+    /// assertion body starting there when it is one character test or an
+    /// alternation of such tests ([`NO_TABLE`] for none; see
+    /// [`quick_asserts`]).
+    quick_of: Vec<u32>,
+    /// Each quick assertion body's slice of `quick_tests`.
+    quick_ranges: Vec<(u32, u32)>,
+    quick_tests: Vec<QuickTest>,
     /// Whether the pattern was a `str` (`Some(true)`) or a bytes-like
     /// object (`Some(false)`); `None` for a pattern compiled from a parsed
     /// tree (`re.Scanner`), which, like CPython's, accepts either subject.
@@ -200,11 +208,15 @@ fn publish(idx: usize, cc: &'static CompiledCode) {
 /// Register compiled code; the handle indexes the registry.
 pub(crate) fn register_code(code: Vec<u32>, groups: usize, is_str: Option<bool>) -> i64 {
     let (table_of, tables) = set_tables(&code);
+    let (quick_of, quick_ranges, quick_tests) = quick_asserts(&code);
     let cc: &'static CompiledCode = Box::leak(Box::new(CompiledCode {
         code,
         groups,
         table_of,
         tables,
+        quick_of,
+        quick_ranges,
+        quick_tests,
         is_str,
     }));
     let mut reg = registry().lock();
@@ -214,6 +226,90 @@ pub(crate) fn register_code(code: Vec<u32>, groups: usize, is_str: Option<bool>)
     idx as i64
 }
 
+/// One test of a quick assertion body (see [`quick_asserts`]).
+#[derive(Clone, Copy)]
+enum QuickTest {
+    /// The character at the position meets this requirement.
+    Char(TailFirst),
+    /// The position is this `AT` position.
+    At(u32),
+}
+
+/// The assertion bodies in `code` that test one character (a literal or
+/// a set, possibly case-folded) or a position (`AT`), or an alternation
+/// of such tests, as lists the matcher evaluates in place instead of
+/// entering itself (such a body sets no marks): textwrap's word splitter
+/// asserts this way at every character (`(?=\s|\Z)`,
+/// `(?<=[\w!"'&.,?])`). As for [`set_tables`], every word equal to an
+/// assertion opcode is tried; the matcher only consults real bodies.
+fn quick_asserts(code: &[u32]) -> (Vec<u32>, Vec<(u32, u32)>, Vec<QuickTest>) {
+    // One test at `op`, and the op after it.
+    let one = |op: usize| -> Option<(QuickTest, usize)> {
+        match *code.get(op)? {
+            OP_AT => Some((QuickTest::At(*code.get(op + 1)?), op + 2)),
+            OP_LITERAL | OP_LITERAL_IGNORE | OP_LITERAL_UNI_IGNORE => {
+                code.get(op + 1)?;
+                Some((QuickTest::Char(Matcher::<u8>::item_first(code, op)), op + 2))
+            }
+            OP_IN | OP_IN_IGNORE | OP_IN_UNI_IGNORE => {
+                let skip = *code.get(op + 1)? as usize;
+                if skip < 2 || code.get(op + 1 + skip - 1) != Some(&OP_FAILURE) {
+                    return None;
+                }
+                Some((
+                    QuickTest::Char(Matcher::<u8>::item_first(code, op)),
+                    op + 1 + skip,
+                ))
+            }
+            _ => None,
+        }
+    };
+    let body_tests = |body: usize| -> Option<Vec<QuickTest>> {
+        if *code.get(body)? != OP_BRANCH {
+            let (t, next) = one(body)?;
+            return (*code.get(next)? == OP_SUCCESS).then(|| vec![t]);
+        }
+        // `<BRANCH> (<skip> <test> <JUMP> <to tail>)* <0>`, the tail the
+        // body's end.
+        let mut tests = Vec::new();
+        let mut alt = body + 1;
+        let mut tail = None;
+        while *code.get(alt)? != 0 {
+            let (t, next) = one(alt + 1)?;
+            if *code.get(next)? != OP_JUMP {
+                return None;
+            }
+            let to = next + 1 + *code.get(next + 1)? as usize;
+            if tail.is_some_and(|t| t != to) {
+                return None;
+            }
+            tail = Some(to);
+            tests.push(t);
+            alt += *code.get(alt)? as usize;
+        }
+        (*code.get(tail?)? == OP_SUCCESS).then_some(tests)
+    };
+    let mut quick_of = vec![NO_TABLE; code.len()];
+    let mut ranges = Vec::new();
+    let mut tests = Vec::new();
+    for (i, &op) in code.iter().enumerate() {
+        // <ASSERT|ASSERT_NOT> <skip> <back> <body...>
+        if !matches!(op, OP_ASSERT | OP_ASSERT_NOT) {
+            continue;
+        }
+        let body = i + 3;
+        if body >= code.len() || quick_of[body] != NO_TABLE {
+            continue;
+        }
+        if let Some(list) = body_tests(body) {
+            quick_of[body] = ranges.len() as u32;
+            ranges.push((tests.len() as u32, (tests.len() + list.len()) as u32));
+            tests.extend(list);
+        }
+    }
+    (quick_of, ranges, tests)
+}
+
 const NO_TABLE: u32 = u32::MAX;
 
 /// Membership tables over the first 256 code points for the character
@@ -659,6 +755,7 @@ struct Matcher<'a, C: SreChar> {
     /// The compiled code's set tables (see [`set_tables`]).
     table_of: &'a [u32],
     tables: &'a [[u64; 4]],
+    cc: &'a CompiledCode,
     beginning: usize,
     start: usize,
     end: usize,
@@ -786,6 +883,7 @@ impl<'a, C: SreChar> Matcher<'a, C> {
             code: &cc.code,
             table_of: &cc.table_of,
             tables: &cc.tables,
+            cc,
             beginning: 0,
             start: 0,
             end: s.len(),
@@ -1232,6 +1330,26 @@ impl<'a, C: SreChar> Matcher<'a, C> {
         self.can_start(self.tail_first(body), pos)
     }
 
+    /// The verdict at `pos` of the assertion body at `body` when it is a
+    /// quick one (see [`quick_asserts`]); `None` for any other body.
+    #[inline]
+    fn quick_assert(&self, body: usize, pos: usize) -> Option<bool> {
+        let cc = self.cc;
+        let at = *cc.quick_of.get(body)?;
+        if at == NO_TABLE {
+            return None;
+        }
+        let (from, to) = cc.quick_ranges[at as usize];
+        Some(
+            cc.quick_tests[from as usize..to as usize]
+                .iter()
+                .any(|t| match *t {
+                    QuickTest::Char(first) => self.simple_can_start(first, pos),
+                    QuickTest::At(code) => self.at(pos, code),
+                }),
+        )
+    }
+
     /// The requirement of the single-character op at `op`.
     #[inline(always)]
     fn item_first(code: &[u32], op: usize) -> TailFirst {
@@ -1966,8 +2084,14 @@ impl<'a, C: SreChar> Matcher<'a, C> {
                     if ptr - self.beginning < back {
                         return Ok(false);
                     }
-                    self.ptr = ptr - back;
-                    if !self.do_match(pat + 2, false)? {
+                    let matched = match self.quick_assert(pat + 2, ptr - back) {
+                        Some(matched) => matched,
+                        None => {
+                            self.ptr = ptr - back;
+                            self.do_match(pat + 2, false)?
+                        }
+                    };
+                    if !matched {
                         return Ok(false);
                     }
                     pat += skip;
@@ -1976,10 +2100,16 @@ impl<'a, C: SreChar> Matcher<'a, C> {
                     let skip = code[pat] as usize;
                     let back = code[pat + 1] as usize;
                     if ptr - self.beginning >= back {
-                        self.ptr = ptr - back;
-                        let save = self.snapshot();
-                        let matched = self.do_match(pat + 2, false)?;
-                        self.restore(&save);
+                        let matched = match self.quick_assert(pat + 2, ptr - back) {
+                            Some(matched) => matched,
+                            None => {
+                                self.ptr = ptr - back;
+                                let save = self.snapshot();
+                                let matched = self.do_match(pat + 2, false)?;
+                                self.restore(&save);
+                                matched
+                            }
+                        };
                         if matched {
                             return Ok(false);
                         }
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index ee22275a..c834ba11 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -156,6 +156,9 @@ pub(crate) enum Verdict {
     /// The class reduces as `copyreg._reduce_newobj` would for a plain
     /// instance (see `Interpreter::reduce_newobj_plain`); `name` unused.
     PlainNewobj = 2,
+    /// A builtin class's own dict has no class method `name` (see
+    /// `Interpreter::leaf_load_type_attr`).
+    NoNativeClassMethod = 3,
 }
 
 pub struct AttrVersion(std::sync::atomic::AtomicU64);
diff --git a/tests/regrtest/test_regex_quick_asserts.py b/tests/regrtest/test_regex_quick_asserts.py
new file mode 100644
index 00000000..2a89f506
--- /dev/null
+++ b/tests/regrtest/test_regex_quick_asserts.py
@@ -0,0 +1,65 @@
+"""Lookarounds over one character test, and `str.translate` tables.
+
+The matcher answers an assertion whose body tests one character (a
+literal or a set, possibly case-folded) or a position, or an alternation
+of such tests, without entering itself. Lookaheads and lookbehinds,
+negated or not, at the subject's edges, and mixed with bodies it cannot
+answer that way must agree with the full matcher. `str.translate` looks
+each ASCII character up once per call.
+"""
+
+import re
+import textwrap
+import unittest
+
+
+class QuickAssertTest(unittest.TestCase):
+    def check(self, pattern, text, expected):
+        self.assertEqual(re.findall(pattern, text), expected, pattern)
+
+    def test_single_tests(self):
+        self.check(r"a(?=b)", "ab ac a", ["a"])
+        self.check(r"a(?!b)", "ab ac a", ["a", "a"])
+        self.check(r"(?<=-)\w", "a-b c-d -", ["b", "d"])
+        self.check(r"(?<!-)\b\w", "a-b c-d", ["a", "c"])
+        self.check(r"x(?=[\s,])", "x x,x.x", ["x", "x"])
+        self.check(r"(?i:x(?=Y))", "xy xY Xy", ["x", "x", "X"])
+        self.check(r"(?<=[^\d\W])\d", "a1 21 _3", ["1", "3"])
+
+    def test_alternations_and_edges(self):
+        self.check(r"\w+(?=\s|\Z)", "one two three", ["one", "two", "three"])
+        self.check(r"\w+(?=,|$)", "a,b c", ["a", "c"])
+        self.check(r"(?<=\s|-)\w", "ab cd-e", ["c", "e"])
+        self.check(r"\w(?!\s|\Z)", "ab c", ["a"])
+        self.check(r"(?<=a|b)c", "ac bc cc", ["c", "c"])
+        self.check(r"(?<!a|b)c", "ac bc cc c", ["c", "c", "c"])
+        self.check(r"(?=a|bc)\w", "a bc bd", ["a", "b"])
+        self.check(r"(?=(a))\w", "ab", ["a"])
+        self.assertEqual(re.split(r"(?<=,)", "a,b,,c"), ["a,", "b,", ",", "c"])
+        self.assertEqual(re.sub(r"(?<=\d)(?=\d)", "_", "1234 5"), "1_2_3_4 5")
+
+    def test_textwrap(self):
+        text = "The well-known quick--brown fox jumps over-the lazy dog. " * 3
+        lines = textwrap.wrap(text, 17)
+        self.assertEqual(
+            lines,
+            ["The well-known", "quick--brown fox", "jumps over-the", "lazy dog. The",
+             "well-known quick", "--brown fox jumps", "over-the lazy", "dog. The well-",
+             "known quick--", "brown fox jumps", "over-the lazy", "dog."],
+        )
+        self.assertEqual(
+            textwrap.TextWrapper(width=8).wrap("a-b-c-d-e x--y"),
+            ["a-b-c-", "d-e x--y"],
+        )
+
+    def test_translate(self):
+        table = {ord("\t"): " ", ord("a"): "AA", ord("b"): None, 0xE9: 0x45}
+        for _ in range(200):
+            self.assertEqual("a\tbécab".translate(table), "AA EcAA")
+        self.assertEqual("abc".translate(str.maketrans("ab", "xy")), "xyc")
+        self.assertEqual("abc".translate([None, None] * 50), "")
+        self.assertEqual("abc\u00e9".translate(list(range(98))), "abc\u00e9")
+
+
+if __name__ == "__main__":
+    unittest.main()

From 7be56351e9ade18310c5ded376624b3529e8f191 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 08:45:43 -0700
Subject: [PATCH 063/173] perf: fixed-point float formatting in integer
 arithmetic

'%.Nf' and format(x, '.Nf') went through Rust's exact float formatter,
whose Grisu path falls back to bignum Dragon for most short precisions.
For at most 17 fractional digits of a magnitude below 2**63, the float
is m * 2**e exactly, so x * 10**N is m * 10**N shifted by e: computed in
128-bit integers and rounded half to even on the exact remainder, the
same correctly rounded digits, about a third of the instructions.

A builtin class's native class method (int.from_bytes) is found through
the type cache rather than a hashed probe of its dict.
---
 crates/weavepy-vm/src/lib.rs              | 84 ++++++++++++++++++++---
 tests/regrtest/test_float_fixed_format.py | 52 ++++++++++++++
 2 files changed, 126 insertions(+), 10 deletions(-)
 create mode 100644 tests/regrtest/test_float_fixed_format.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 693a7a80..97a999f3 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -27282,19 +27282,25 @@ impl Interpreter {
         if cls.flags.is_builtin
             && !cls.memo_verdict(crate::types::Verdict::NoNativeClassMethod, name, || {
                 cls.dict.try_borrow().is_ok_and(|d| {
-                    !matches!(d.get(&crate::object::StrKey(name)), Some(Object::ClassMethod(_)))
+                    !matches!(
+                        d.get(&crate::object::StrKey(name)),
+                        Some(Object::ClassMethod(_))
+                    )
                 })
             })
         {
-            let d = cls.dict.try_borrow().ok()?;
-            if let Some(Object::ClassMethod(cm)) = d.get(&crate::object::StrKey(name)) {
-                let f = cm.func();
-                return matches!(f, Object::Builtin(_)).then(|| {
-                    Object::BoundMethod(Rc::new(BoundMethod::py_method(
-                        Object::Type(cls.clone()),
-                        f,
-                    )))
-                });
+            // (Through the type cache: the class's own entry is the one
+            // its MRO walk finds first.)
+            if let Some((Object::ClassMethod(cm), owner)) = cls.lookup_with_owner(name) {
+                if Rc::ptr_eq(&owner, cls) {
+                    let f = cm.func();
+                    return matches!(f, Object::Builtin(_)).then(|| {
+                        Object::BoundMethod(Rc::new(BoundMethod::py_method(
+                            Object::Type(cls.clone()),
+                            f,
+                        )))
+                    });
+                }
             }
         }
         match Self::leaf_class_attr(code, cls, name_idx)? {
@@ -70734,6 +70740,9 @@ fn format_bigint_hex(b: &num_bigint::BigInt, upper: bool, p: &ParsedSpec) -> Str
 /// rest (`'%.123456f'` matches CPython instead of aborting).
 fn fixed_core(mag: f64, prec: usize) -> String {
     const FIXED_PREC_CAP: usize = 1100;
+    if let Some(s) = fixed_core_word(mag, prec) {
+        return s;
+    }
     if prec <= FIXED_PREC_CAP {
         format!("{mag:.*}", prec)
     } else {
@@ -70744,6 +70753,61 @@ fn fixed_core(mag: f64, prec: usize) -> String {
     }
 }
 
+/// [`fixed_core`] in 128-bit integer arithmetic, for the common shapes:
+/// at most 17 fractional digits of a magnitude below 2**63. The float is
+/// `m * 2**e` exactly, so `mag * 10**prec` is `m * 10**prec` shifted by
+/// `e`, rounded half to even on the exact remainder (the correctly
+/// rounded answer the general formatter gives). `None` for any other
+/// magnitude or precision.
+fn fixed_core_word(mag: f64, prec: usize) -> Option<String> {
+    if prec > 17 {
+        return None;
+    }
+    let bits = mag.to_bits();
+    let exp_field = ((bits >> 52) & 0x7ff) as i32;
+    let frac = u128::from(bits & ((1u64 << 52) - 1));
+    let (m, e) = if exp_field == 0 {
+        (frac, -1074)
+    } else {
+        (frac | 1 << 52, exp_field - 1075)
+    };
+    let p10 = 10u128.pow(prec as u32);
+    let scaled = if e >= 0 {
+        // `m << e` stays below 2**63, and times `10**17` below 2**120.
+        if e > 10 {
+            return None;
+        }
+        (m << e) * p10
+    } else if -e > 120 {
+        // Below 2**-68 the scaled value is under one half: it rounds to 0.
+        0
+    } else {
+        let sh = e.unsigned_abs();
+        let num = m * p10;
+        let q = num >> sh;
+        let r = num & ((1u128 << sh) - 1);
+        let half = 1u128 << (sh - 1);
+        q + u128::from(r > half || (r == half && q & 1 == 1))
+    };
+    let digits = scaled.to_string();
+    if prec == 0 {
+        return Some(digits);
+    }
+    let mut out = String::with_capacity(digits.len().max(prec + 1) + 1);
+    if digits.len() <= prec {
+        out.push('0');
+        out.push('.');
+        out.extend(std::iter::repeat_n('0', prec - digits.len()));
+        out.push_str(&digits);
+    } else {
+        let (int, frac) = digits.split_at(digits.len() - prec);
+        out.push_str(int);
+        out.push('.');
+        out.push_str(frac);
+    }
+    Some(out)
+}
+
 /// `true` when every decimal digit in `core` is `'0'` (and there is at
 /// least one). Used to decide PEP 682 negative-zero coercion: a value that
 /// renders with all-zero digits *is* zero at this precision.
diff --git a/tests/regrtest/test_float_fixed_format.py b/tests/regrtest/test_float_fixed_format.py
new file mode 100644
index 00000000..97d54267
--- /dev/null
+++ b/tests/regrtest/test_float_fixed_format.py
@@ -0,0 +1,52 @@
+"""Fixed-point float formatting (`'%.Nf'`, `format(x, '.Nf')`) is
+correctly rounded.
+
+Short precisions of moderate magnitudes are formatted in integer
+arithmetic; the result must equal the exact binary value rounded half to
+even, with the sign kept for negative values that round to zero, at the
+edges of that range and past it.
+"""
+
+import random
+import struct
+import unittest
+from fractions import Fraction
+
+
+def reference(x, prec):
+    q = Fraction(x) * 10**prec
+    n = round(q)  # half to even on the exact value
+    neg = x < 0 or (x == 0 and str(x).startswith("-"))
+    digits = str(abs(n)).rjust(prec + 1, "0")
+    body = digits if prec == 0 else digits[:-prec] + "." + digits[-prec:]
+    return ("-" if neg else "") + body
+
+
+class FixedFormatTest(unittest.TestCase):
+    def test_against_exact_rounding(self):
+        r = random.Random(5)
+        values = [0.0, -0.0, 0.5, 1.5, 2.5, -2.5, 0.125, 1.005, 2.675, 0.45,
+                  1e-17, 5e-18, 5e-324, 2.0**53 + 2, 2.0**62, 2.0**63 - 1024,
+                  2.0**63, 2.0**64, 1e20, -0.04]
+        for _ in range(250):
+            values.append(r.uniform(-1e5, 1e5))
+            values.append(r.randrange(-10**6, 10**6) / r.choice([8, 1024, 10, 1000]))
+            values.append(struct.unpack("<d", struct.pack("<Q", r.getrandbits(64)))[0])
+        for v in values:
+            if v != v or abs(v) == float("inf") or abs(v) > 1e30:
+                continue
+            for prec in (0, 1, 3, 6, 17, 18):
+                self.assertEqual("%.*f" % (prec, v), reference(v, prec), (v, prec))
+                self.assertEqual(format(v, ".%df" % prec), reference(v, prec), (v, prec))
+
+    def test_flags(self):
+        self.assertEqual("%8.3f|%-8.2f|%+.1f" % (1.5, -2.25, 0.05), "   1.500|-2.25   |+0.1")
+        self.assertEqual(format(-0.0001, "z.2f"), "0.00")
+        self.assertEqual(format(-0.0001, ".2f"), "-0.00")
+        self.assertEqual(format(2.0, "#.0f"), "2.")
+        self.assertEqual(format(1234567.891, ",.2f"), "1,234,567.89")
+        self.assertEqual(format(0.5, "010.3f"), "000000.500")
+
+
+if __name__ == "__main__":
+    unittest.main()

From 581bb09dd046b127ae5f719a72babfde85007e15 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 09:21:28 -0700
Subject: [PATCH 064/173] perf: module_attr off sys.modules, JIT empty dicts
 and temp tuple loops

Interpreter::module_attr reads an imported top-level module's attribute
straight off its sys.modules entry when no thread is initializing it,
instead of walking the dotted import path (a split, string building and
the per-part loader bookkeeping) on every call: object.__reduce_ex__
fetches copyreg.__newobj__ this way for every instance it reduces.

The frame JIT builds an empty dict (BUILD_MAP 0) in native code, and a
FOR_ITER that exhausts the iterator of a uniquely held tuple of ints,
floats, strings, bools or None frees it in place: such items finalize
nothing, so the release needs no trip through the core loop (tomllib's
Flags.is_ loops over key[:-1]).
---
 crates/weavepy-vm/src/frame_jit.rs | 20 +++++++++++++++++---
 crates/weavepy-vm/src/lib.rs       |  9 +++++++++
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 5d37cda0..834f0bcb 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1442,7 +1442,21 @@ unsafe extern "C" fn h_for_iter_rest(
                 // loop's arm retires it), when its release can't free the
                 // tuple (whose items could queue finalizers): a shared
                 // iterator stays exhausted, the tuple immutable.
-                if unique && crate::shared_value::ThinArc::strong_count(items) == 1 {
+                // (Items that finalize nothing free in place: a temporary
+                // tuple of names, `for k in key[:-1]`.)
+                if unique
+                    && crate::shared_value::ThinArc::strong_count(items) == 1
+                    && !items.iter().all(|x| {
+                        matches!(
+                            x,
+                            Object::Int(_)
+                                | Object::Float(_)
+                                | Object::Bool(_)
+                                | Object::None
+                                | Object::Str(_)
+                        )
+                    })
+                {
                     return 3;
                 }
                 // SAFETY: the iterator leaves the stack.
@@ -6139,7 +6153,7 @@ fn native_at(code: &CodeObject, pc: usize) -> bool {
     let ins = code.instructions[pc];
     match ins.op {
         OpCode::BuildTuple => (1..=3).contains(&ins.arg),
-        OpCode::BuildMap => (1..=8).contains(&ins.arg),
+        OpCode::BuildMap => (0..=8).contains(&ins.arg),
         OpCode::UnaryOp => ins.arg <= 3,
         OpCode::RaiseVarargs => matches!(ins.arg, 1 | 2),
         // A prologue `MAKE_CELL`: the activation's cells are built with it.
@@ -7205,7 +7219,7 @@ impl<'a> Lower<'a> {
                 return self.build(pc, ins.arg as usize, false)
             }
             OpCode::BuildList => return self.build(pc, ins.arg as usize, true),
-            OpCode::BuildMap if (1..=8).contains(&ins.arg) => {
+            OpCode::BuildMap if (0..=8).contains(&ins.arg) => {
                 return self.build_map(pc, ins.arg as usize)
             }
             // PEP 709: the local's value (`Unbound` included) moves onto
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 97a999f3..4146af47 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -58471,6 +58471,15 @@ impl Interpreter {
     /// Import `module` and fetch one of its top-level attributes by name.
     /// Returns `None` if the module can't be imported or lacks the name.
     pub(crate) fn module_attr(&mut self, module: &str, attr: &str) -> Option<Object> {
+        // An imported top-level module that no thread is initializing: its
+        // attribute straight off the `sys.modules` entry (what `load_one`
+        // returns for it), without the dotted walk's bookkeeping.
+        // `object.__reduce_ex__` reads `copyreg.__newobj__` this way.
+        if !module.contains('.') && self.cache.initializing_holder(module).is_none() {
+            if let Some(Object::Module(m)) = self.cache.get(module) {
+                return m.dict.borrow().get(&crate::object::StrKey(attr)).cloned();
+            }
+        }
         let m = self.import_path(module).ok()?;
         if let Object::Module(m) = m {
             return m.dict.borrow().get(&crate::object::StrKey(attr)).cloned();

From e73327370fbf6731e66ff9340f6a84406a661fba Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 09:53:39 -0700
Subject: [PATCH 065/173] perf: keep the regex matcher's AT test in line

Matcher::at gained a second caller with the quick assertions, and the
compiler stopped inlining it into the matcher's AT arm: every \b and ^
became a call. Marking it always-inline restores the arm (regex_mix
drops 1.5%).
---
 crates/weavepy-vm/src/stdlib/sre_mod.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/crates/weavepy-vm/src/stdlib/sre_mod.rs b/crates/weavepy-vm/src/stdlib/sre_mod.rs
index de632e05..48bcdab5 100644
--- a/crates/weavepy-vm/src/stdlib/sre_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/sre_mod.rs
@@ -1129,6 +1129,9 @@ impl<'a, C: SreChar> Matcher<'a, C> {
         r
     }
 
+    // (In line in the matcher's `AT` arm, which a second caller, the
+    // quick assertions, would otherwise talk the compiler out of.)
+    #[inline(always)]
     fn at(&self, ptr: usize, atcode: u32) -> bool {
         let s = self.s;
         match atcode {

From 785d1aa13f9902f12d132c468eff658c7b211a47 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 10:27:33 -0700
Subject: [PATCH 066/173] perf: keep os.PathLike's structural check on
 isinstance misses

The isinstance leaf answered a miss on a plain instance as final, but
os.PathLike is checked structurally (any class defining __fspath__),
on the full path only. A miss against it now goes there again, which
restores tempfile's and pathlib-style FakePath checks (test_tempfile,
test_os, test_zipfile and others failed).
---
 crates/weavepy-vm/src/lib.rs                      |  6 ++++--
 tests/regrtest/test_isinstance_class_attr_miss.py | 12 ++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 4146af47..b86f5233 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -18826,8 +18826,10 @@ impl Interpreter {
             if ic.is_subclass_of(cls) {
                 return Some(Ok(Object::Bool(true)));
             }
-            // A miss is final when `__class__` can't answer otherwise.
-            return ic.class_attr_is_plain().then_some(Ok(Object::Bool(false)));
+            // A miss is final when `__class__` can't answer otherwise (and
+            // the class isn't `os.PathLike`, whose check is structural).
+            return (ic.class_attr_is_plain() && !crate::stdlib::os::is_path_like(cls))
+                .then_some(Ok(Object::Bool(false)));
         }
         // One class, or a flat tuple of them (`isinstance(x, (str, int))`
         // asks each in turn).
diff --git a/tests/regrtest/test_isinstance_class_attr_miss.py b/tests/regrtest/test_isinstance_class_attr_miss.py
index 60248d6f..fe9082f6 100644
--- a/tests/regrtest/test_isinstance_class_attr_miss.py
+++ b/tests/regrtest/test_isinstance_class_attr_miss.py
@@ -64,6 +64,18 @@ def getattribute(self, name):
         for _ in range(2000):
             self.assertFalse(isinstance(c, float))
 
+    def test_structural_pathlike(self):
+        import os
+
+        class Fake:
+            def __fspath__(self):
+                return b"/"
+
+        f = Fake()
+        for _ in range(2000):
+            self.assertTrue(isinstance(f, os.PathLike))
+            self.assertFalse(isinstance(Plain(), os.PathLike))
+
     def test_class_assignment(self):
         class A:
             pass

From 5ca81fda0fd83f49ecf4deb327c5f5cc1c06e677 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 09:04:32 -0700
Subject: [PATCH 067/173] perf: pool the identifiers in code objects' name
 tables

A code object's co_names, co_varnames, co_freevars and co_cellvars held
one String per entry, and a module's code objects repeat the same few
identifiers: importing asyncio decodes about 38,000 entries, of which
about 6,300 are distinct, at 24 bytes plus a heap block each.

The tables now hold `Name`s, shared immutable strings from one
process-wide pool (as CPython interns identifiers), so equal names share
one allocation and an entry costs 16 bytes. The pool sweeps names no
code object uses any more whenever it has doubled since its last sweep.

The code cache's native form (version 5) writes each of a module's
identifiers once and refers to it by number afterwards, so the reader
decodes and pools each distinct identifier once per module rather than
allocating a string per entry.

Peak RSS: import asyncio 28.12 -> 27.73 MB, import re 15.29 -> 15.01 MB,
import typing, dataclasses, enum 21.88 -> 21.61 MB; instructions within
noise.
---
 crates/weavepy-capi/src/abi314.rs             |   4 +-
 crates/weavepy-compiler/src/cpython_code.rs   |  22 +-
 crates/weavepy-compiler/src/intern.rs         |   4 +-
 crates/weavepy-compiler/src/lib.rs            | 104 ++++---
 crates/weavepy-compiler/src/name.rs           | 260 ++++++++++++++++++
 crates/weavepy-compiler/src/native_code.rs    |  75 +++--
 crates/weavepy-jit/src/analyze.rs             |  16 +-
 crates/weavepy-vm/src/builtins.rs             |  23 +-
 crates/weavepy-vm/src/lib.rs                  |  14 +-
 crates/weavepy-vm/src/object.rs               |   4 +-
 crates/weavepy-vm/src/specialize.rs           |   2 +-
 .../src/stdlib/elementtree_native.rs          |   2 +-
 crates/weavepy-vm/src/stdlib/marshal_mod.rs   |  36 ++-
 crates/weavepy-vm/src/tier2.rs                |   9 +-
 14 files changed, 459 insertions(+), 116 deletions(-)
 create mode 100644 crates/weavepy-compiler/src/name.rs

diff --git a/crates/weavepy-capi/src/abi314.rs b/crates/weavepy-capi/src/abi314.rs
index 1382da80..4183cb71 100644
--- a/crates/weavepy-capi/src/abi314.rs
+++ b/crates/weavepy-capi/src/abi314.rs
@@ -1574,13 +1574,13 @@ unsafe fn code_names(
 /// `PyCode_GetCellvars(code)` — new reference to `co_cellvars`.
 #[no_mangle]
 pub unsafe extern "C" fn PyCode_GetCellvars(code: *mut PyObject) -> *mut PyObject {
-    unsafe { code_names(code, |c| c.cellvars.clone()) }
+    unsafe { code_names(code, |c| c.cellvars.iter().map(String::from).collect()) }
 }
 
 /// `PyCode_GetFreevars(code)` — new reference to `co_freevars`.
 #[no_mangle]
 pub unsafe extern "C" fn PyCode_GetFreevars(code: *mut PyObject) -> *mut PyObject {
-    unsafe { code_names(code, |c| c.freevars.clone()) }
+    unsafe { code_names(code, |c| c.freevars.iter().map(String::from).collect()) }
 }
 
 // ---------------------------------------------------------------------------
diff --git a/crates/weavepy-compiler/src/cpython_code.rs b/crates/weavepy-compiler/src/cpython_code.rs
index 266edc81..7a884f04 100644
--- a/crates/weavepy-compiler/src/cpython_code.rs
+++ b/crates/weavepy-compiler/src/cpython_code.rs
@@ -731,7 +731,7 @@ pub const CO_FAST_FREE: u8 = 0x80;
 /// `CO_FAST_LOCAL|CO_FAST_CELL` rather than getting its own entry;
 /// the argument slots carry their `CO_FAST_ARG_*` kind and hidden
 /// comprehension locals `CO_FAST_HIDDEN`.
-fn build_localsplus(code: &CodeObject) -> (Vec<String>, Vec<u8>) {
+fn build_localsplus(code: &CodeObject) -> (Vec<crate::Name>, Vec<u8>) {
     let mut names = Vec::with_capacity(code.varnames.len() + code.cellvars.len());
     let mut kinds = Vec::with_capacity(names.capacity());
     // `argvarkinds`: pos-only, pos-or-kw, kw-only, *args, **kwargs.
@@ -1053,7 +1053,7 @@ pub fn encode(code: &CodeObject) -> CpythonCode {
         co_linetable: encode_linetable(code, &ext, &mapped, &zero_width, firstlineno),
         co_exceptiontable: encode_exception_table(code, &starts),
         co_code,
-        localsplusnames,
+        localsplusnames: localsplusnames.into_iter().map(String::from).collect(),
         localspluskinds,
         stacksize: code.stacksize.unwrap_or_else(|| compute_stacksize(code)),
         firstlineno,
@@ -2061,7 +2061,7 @@ impl SlotMap {
     /// Build from a code object's variable lists (mirrors
     /// [`DerefSlots::from_code`], inverted).
     #[must_use]
-    pub fn from_code_vars(varnames: &[String], cellvars: &[String], freevars: &[String]) -> Self {
+    pub fn from_code_vars<S: PartialEq>(varnames: &[S], cellvars: &[S], freevars: &[S]) -> Self {
         let nlocals = varnames.len() as u32;
         let mut nslots = varnames.len() + freevars.len();
         for c in cellvars {
@@ -2813,7 +2813,7 @@ mod tests {
             ..CodeObject::default()
         };
         // Give a couple of locals so LOAD_FAST vs LOAD_CLOSURE disambiguates.
-        c.varnames = vec!["a".to_owned(), "b".to_owned()];
+        c.varnames = vec!["a".into(), "b".into()];
         c
     }
 
@@ -3110,9 +3110,9 @@ mod tests {
             linetable: vec![1, 2, 2, 3, 3, 4, 4].into(),
             ..CodeObject::default()
         };
-        code.varnames = vec!["a".to_owned(), "b".to_owned()];
-        code.cellvars = vec!["c".to_owned()];
-        code.freevars = vec!["f".to_owned()];
+        code.varnames = vec!["a".into(), "b".into()];
+        code.cellvars = vec!["c".into()];
+        code.freevars = vec!["f".into()];
         code.exception_table.push(ExcHandler {
             start: 1,
             end: 4,
@@ -3147,9 +3147,9 @@ mod tests {
             linetable: dc.linetable.into(),
             ..CodeObject::default()
         };
-        code2.varnames = dc.varnames;
-        code2.cellvars = dc.cellvars;
-        code2.freevars = dc.freevars;
+        code2.varnames = dc.varnames.iter().map(crate::Name::from).collect();
+        code2.cellvars = dc.cellvars.iter().map(crate::Name::from).collect();
+        code2.freevars = dc.freevars.iter().map(crate::Name::from).collect();
         code2.exception_table = dc.exception_table;
         let cp2 = encode(&code2);
         assert_eq!(cp2.co_code, cp.co_code);
@@ -3315,7 +3315,7 @@ def f():
 /// mapping pattern under-reported by one).
 #[test]
 fn stack_effects_agree_with_wire_shape() {
-    let slots = SlotMap::from_code_vars(&[], &[], &[]);
+    let slots = SlotMap::from_code_vars::<crate::Name>(&[], &[], &[]);
     // Opcodes whose WeavePy arg is not the wire arg, or whose
     // fallthrough effect legitimately differs from the wire shape
     // (jumps take their real effect on the edge; FOR_ITER/END_FOR
diff --git a/crates/weavepy-compiler/src/intern.rs b/crates/weavepy-compiler/src/intern.rs
index 02d40507..c6750fa1 100644
--- a/crates/weavepy-compiler/src/intern.rs
+++ b/crates/weavepy-compiler/src/intern.rs
@@ -42,7 +42,7 @@ impl InternIndex {
         i
     }
 
-    pub(crate) fn intern_name(&mut self, values: &mut Vec<String>, name: &str) -> u32 {
+    pub(crate) fn intern_name(&mut self, values: &mut Vec<crate::Name>, name: &str) -> u32 {
         if values.len() < 32 {
             if let Some(i) = values.iter().position(|v| v == name) {
                 return i as u32;
@@ -55,7 +55,7 @@ impl InternIndex {
             self.next.push(self.heads.insert(key, values.len() as u32));
         }
         let i = values.len() as u32;
-        values.push(name.to_owned());
+        values.push(crate::Name::new(name));
         i
     }
 
diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs
index 48003585..a55917ba 100644
--- a/crates/weavepy-compiler/src/lib.rs
+++ b/crates/weavepy-compiler/src/lib.rs
@@ -40,6 +40,7 @@ pub mod cpython_code;
 mod flowgraph;
 mod intern;
 mod mangle;
+mod name;
 pub mod native_code;
 mod validate;
 
@@ -50,6 +51,7 @@ pub use bytecode::{
     COMPARE_OP_TO_BOOL_FLAG, COOLDOWN, SPECIAL_AENTER, SPECIAL_AEXIT, SPECIAL_ENTER, SPECIAL_EXIT,
 };
 pub use cpython_code::{CpythonCode, Position};
+pub use name::Name;
 
 /// CPython compile.c `STACK_USE_GUIDELINE`: literal displays and call
 /// sites with more operands than this compile through accumulator
@@ -145,6 +147,11 @@ pub use weavepy_parser::ast::expr_name;
 
 // ---------- code object ----------
 
+/// Whether `names` holds `name`.
+fn has_name(names: &[Name], name: &str) -> bool {
+    names.iter().any(|n| n.as_str() == name)
+}
+
 /// RFC 0061 (WS2a): an opaque, VM-owned per-code-object extension slot.
 ///
 /// The VM stashes derived, execution-only state here (today: the
@@ -597,13 +604,14 @@ pub struct CodeObject {
     pub jit_hint: JitHint,
     pub constants: Vec<Constant>,
     /// Names referenced by `LOAD_NAME` / `LOAD_GLOBAL` / `STORE_NAME` etc.
-    pub names: Vec<String>,
+    /// These and the other name tables hold pooled [`Name`]s.
+    pub names: Vec<Name>,
     /// Local variable names (positional + keyword + `*args`/`**kwargs` + locals).
-    pub varnames: Vec<String>,
+    pub varnames: Vec<Name>,
     /// Free variables — read from an enclosing scope.
-    pub freevars: Vec<String>,
+    pub freevars: Vec<Name>,
     /// Cell variables — locally defined but referenced by an inner scope.
-    pub cellvars: Vec<String>,
+    pub cellvars: Vec<Name>,
     /// Out-of-line exception handlers. Looked up by current PC when a
     /// `RuntimeError::PyException` propagates through this code object.
     pub exception_table: Vec<ExcHandler>,
@@ -1226,7 +1234,7 @@ impl CodeObject {
                     }
                 }
                 OpCode::LoadDeref | OpCode::StoreDeref | OpCode::LoadClosure => {
-                    let combined: Vec<&String> =
+                    let combined: Vec<&Name> =
                         self.cellvars.iter().chain(self.freevars.iter()).collect();
                     if let Some(n) = combined.get(ins.arg as usize) {
                         out.push('(');
@@ -2865,10 +2873,10 @@ impl Compiler {
         // cell index. Keeping the internal order equal to slot order
         // makes the RFC 0068 wire codec's deref mapping invertible.
         if self.co.cellvars.len() > 1 {
-            let mut sorted: Vec<String> = self.co.cellvars.clone();
+            let mut sorted: Vec<Name> = self.co.cellvars.clone();
             sorted.sort_unstable_by_key(|c| match self.co.varnames.iter().position(|v| v == c) {
                 Some(p) => (0usize, p, String::new()),
-                None => (1usize, 0, c.clone()),
+                None => (1usize, 0, c.to_string()),
             });
             if sorted != self.co.cellvars {
                 let remap: Vec<u32> = self
@@ -2928,7 +2936,7 @@ impl Compiler {
                 self.free_order = sorted;
             }
         }
-        self.co.freevars = self.free_order.clone();
+        self.co.freevars = self.free_order.iter().map(Name::from).collect();
 
         // `codegen_wrap_in_stopiteration_handler` (PEP 479): the
         // generator-family body ends in an explicit `return None`, and
@@ -3559,7 +3567,7 @@ impl Compiler {
         if has_annotations {
             self.co
                 .cellvars
-                .push("__conditional_annotations__".to_owned());
+                .push(Name::from("__conditional_annotations__"));
             if let Some(loc) = module_loc {
                 self.apply_annotate_loc(loc);
             }
@@ -3731,10 +3739,10 @@ impl Compiler {
                 _ => {}
             }
         }
-        let mut all: Vec<String> = self.co.cellvars.clone();
+        let mut all: Vec<Name> = self.co.cellvars.clone();
         for name in comp_cells {
-            if !all.contains(&name) {
-                all.push(name);
+            if !all.iter().any(|c| *c == name) {
+                all.push(Name::from(name));
             }
         }
         all.sort_unstable();
@@ -3846,8 +3854,8 @@ impl Compiler {
         for name in needed_in_inner {
             if matches!(self.bindings.get(&name), Some(Binding::Local)) {
                 self.bindings.insert(name.clone(), Binding::Cell);
-                if !self.co.cellvars.contains(&name) {
-                    self.co.cellvars.push(name);
+                if !has_name(&self.co.cellvars, &name) {
+                    self.co.cellvars.push(Name::from(name));
                 }
             }
         }
@@ -5769,7 +5777,7 @@ impl Compiler {
         }
         inner.co.qualname = self.compute_child_qualname(hidden_name);
         inner.co.arg_count = arg_count;
-        inner.co.varnames = hidden_params.to_vec();
+        inner.co.varnames = hidden_params.iter().map(Name::from).collect();
         inner.current_line = entry_line.unwrap_or(self.current_line);
         if class_scope {
             // `.type_params` is read by the class body (always a cell,
@@ -5778,7 +5786,10 @@ impl Compiler {
             inner
                 .bindings
                 .insert(".type_params".to_owned(), Binding::Cell);
-            inner.co.cellvars.push(".type_params".to_owned());
+            inner
+                .co
+                .cellvars
+                .push(Name::from(".type_params".to_owned()));
             inner
                 .bindings
                 .insert(".generic_base".to_owned(), Binding::Local);
@@ -5787,8 +5798,8 @@ impl Compiler {
         for free in &inner.free_order {
             if matches!(self.bindings.get(free), Some(Binding::Local)) {
                 self.bindings.insert(free.clone(), Binding::Cell);
-                if !self.co.cellvars.contains(free) {
-                    self.co.cellvars.push(free.clone());
+                if !has_name(&self.co.cellvars, free) {
+                    self.co.cellvars.push(Name::from(free.clone()));
                 }
             }
         }
@@ -6254,7 +6265,7 @@ impl Compiler {
         inner.co.kwonly_count = kwonly_count;
         inner.co.has_varargs = args.vararg.is_some();
         inner.co.has_varkeywords = args.kwarg.is_some();
-        inner.co.varnames = param_names.clone();
+        inner.co.varnames = param_names.iter().map(Name::from).collect();
         inner.current_line = entry_line.unwrap_or(self.current_line);
         // Methods compiled inside a class body get an implicit
         // `__class__` free variable so `super()` (and explicit
@@ -6347,8 +6358,8 @@ impl Compiler {
         for free in &inner.free_order {
             if matches!(self.bindings.get(free), Some(Binding::Local)) {
                 self.bindings.insert(free.clone(), Binding::Cell);
-                if !self.co.cellvars.contains(free) {
-                    self.co.cellvars.push(free.clone());
+                if !has_name(&self.co.cellvars, free) {
+                    self.co.cellvars.push(Name::from(free.clone()));
                 }
             }
         }
@@ -6512,14 +6523,14 @@ impl Compiler {
             Some((String::new(), String::new(), self.annotation_child_prefix()));
         inner.co.arg_count = 1;
         inner.co.posonly_count = 1;
-        inner.co.varnames = vec![FORMAT_PARAM.to_owned()];
+        inner.co.varnames = vec![Name::from(FORMAT_PARAM)];
         inner.current_line = loc.line;
         inner.analyze_scope_function(&[FORMAT_PARAM.to_owned()], analysis_body, &[&self.bindings]);
         for free in &inner.free_order {
             if matches!(self.bindings.get(free), Some(Binding::Local)) {
                 self.bindings.insert(free.clone(), Binding::Cell);
-                if !self.co.cellvars.contains(free) {
-                    self.co.cellvars.push(free.clone());
+                if !has_name(&self.co.cellvars, free) {
+                    self.co.cellvars.push(Name::from(free.clone()));
                 }
             }
         }
@@ -6551,7 +6562,7 @@ impl Compiler {
         let mut code = inner.finish();
         // `co->co_localsplusnames = ("format", *co->co_localsplusnames[1:])`
         if is_annotate {
-            code.varnames[0] = "format".to_owned();
+            code.varnames[0] = Name::from("format");
             code.annotate_scope = true;
         }
         Ok(code)
@@ -7037,7 +7048,7 @@ impl Compiler {
                 || class_body_defs_claim_class_cell(body)
         };
         if needs_class_closure {
-            inner.co.cellvars.push("__class__".to_owned());
+            inner.co.cellvars.push(Name::from("__class__".to_owned()));
             inner.bindings.insert("__class__".to_owned(), Binding::Cell);
         }
         // PEP 695 (RFC 0051): annotation scopes created in this class
@@ -7053,7 +7064,10 @@ impl Compiler {
             .any(|s| stmt_needs_classdict(s, future_annotations))
             || needed.contains("__classdict__");
         if needs_classdict {
-            inner.co.cellvars.push("__classdict__".to_owned());
+            inner
+                .co
+                .cellvars
+                .push(Name::from("__classdict__".to_owned()));
             inner
                 .bindings
                 .insert("__classdict__".to_owned(), Binding::Cell);
@@ -7069,7 +7083,7 @@ impl Compiler {
             inner
                 .co
                 .cellvars
-                .push("__conditional_annotations__".to_owned());
+                .push(Name::from("__conditional_annotations__"));
             inner
                 .bindings
                 .insert("__conditional_annotations__".to_owned(), Binding::Cell);
@@ -7218,7 +7232,7 @@ impl Compiler {
                 inner.collect_comp_cells_stmt(s, &mut comp_cells);
             }
             if !comp_cells.is_empty() {
-                let implicit: Vec<String> = inner
+                let implicit: Vec<Name> = inner
                     .co
                     .cellvars
                     .iter()
@@ -7432,10 +7446,10 @@ impl Compiler {
         let inner_freevars = inner_code.freevars.clone();
 
         for free in &inner_freevars {
-            if matches!(self.bindings.get(free), Some(Binding::Local)) {
-                self.bindings.insert(free.clone(), Binding::Cell);
-                if !self.co.cellvars.contains(free) {
-                    self.co.cellvars.push(free.clone());
+            if matches!(self.bindings.get(free.as_str()), Some(Binding::Local)) {
+                self.bindings.insert(free.to_string(), Binding::Cell);
+                if !has_name(&self.co.cellvars, free) {
+                    self.co.cellvars.push(Name::from(free.clone()));
                 }
             }
         }
@@ -9554,7 +9568,7 @@ impl Compiler {
         if let Some(i) = self.co.varnames.iter().position(|n| n == name) {
             return i as u32;
         }
-        self.co.varnames.push(name.to_owned());
+        self.co.varnames.push(Name::from(name.to_owned()));
         (self.co.varnames.len() - 1) as u32
     }
 
@@ -11507,11 +11521,11 @@ impl Compiler {
                     let pos = self.free_order.iter().position(|n| n == name).unwrap_or(0);
                     (self.co.cellvars.len() + pos) as u32
                 } else {
-                    if !self.co.cellvars.contains(name) {
+                    if !has_name(&self.co.cellvars, name) {
                         // Normally pre-registered by the scope's
                         // analysis (`register_comp_cells`); a late
                         // arrival still gets its slot.
-                        self.co.cellvars.push(name.clone());
+                        self.co.cellvars.push(Name::from(name.clone()));
                     }
                     self.co.cellvars.iter().position(|n| n == name).unwrap_or(0) as u32
                 };
@@ -11686,7 +11700,7 @@ impl Compiler {
         // `compiler_set_qualname` doesn't special-case comprehensions.
         inner.co.qualname = self.compute_child_qualname(name);
         inner.co.arg_count = 1;
-        inner.co.varnames.push(".0".to_owned());
+        inner.co.varnames.push(Name::from(".0".to_owned()));
         inner.bindings.insert(".0".to_owned(), Binding::Local);
         if is_async_comp && !matches!(kind, CompKind::Generator) {
             inner.co.is_coroutine = true;
@@ -11786,8 +11800,8 @@ impl Compiler {
                     _ => {
                         if matches!(enclosing, None | Some(Binding::Local)) {
                             self.bindings.insert(name.clone(), Binding::Cell);
-                            if !self.co.cellvars.contains(&name) {
-                                self.co.cellvars.push(name.clone());
+                            if !has_name(&self.co.cellvars, &name) {
+                                self.co.cellvars.push(Name::from(name.clone()));
                             }
                         }
                         Binding::Free
@@ -11871,8 +11885,8 @@ impl Compiler {
             for name in needed_in_inner {
                 if matches!(inner.bindings.get(&name), Some(Binding::Local)) {
                     inner.bindings.insert(name.clone(), Binding::Cell);
-                    if !inner.co.cellvars.contains(&name) {
-                        inner.co.cellvars.push(name);
+                    if !has_name(&inner.co.cellvars, &name) {
+                        inner.co.cellvars.push(Name::from(name));
                     }
                 }
             }
@@ -11949,10 +11963,10 @@ impl Compiler {
 
         // Promote our locals to cells where needed.
         for free in &inner_freevars {
-            if matches!(self.bindings.get(free), Some(Binding::Local)) {
-                self.bindings.insert(free.clone(), Binding::Cell);
-                if !self.co.cellvars.contains(free) {
-                    self.co.cellvars.push(free.clone());
+            if matches!(self.bindings.get(free.as_str()), Some(Binding::Local)) {
+                self.bindings.insert(free.to_string(), Binding::Cell);
+                if !has_name(&self.co.cellvars, free) {
+                    self.co.cellvars.push(Name::from(free.clone()));
                 }
             }
         }
diff --git a/crates/weavepy-compiler/src/name.rs b/crates/weavepy-compiler/src/name.rs
new file mode 100644
index 00000000..3a623152
--- /dev/null
+++ b/crates/weavepy-compiler/src/name.rs
@@ -0,0 +1,260 @@
+//! Interned identifiers for code objects' name tables.
+//!
+//! A module's code objects repeat the same few identifiers (`self`,
+//! `__name__`, `len`, a class's attribute names) across their `co_names`,
+//! `co_varnames`, `co_freevars` and `co_cellvars`: importing `asyncio`
+//! decodes about 38,000 entries, of which about 6,300 are distinct. A
+//! [`Name`] is a shared, immutable string, and every name the compiler or
+//! the code cache produces goes through one process-wide pool, so equal
+//! identifiers share one allocation (as CPython interns them).
+//!
+//! The pool holds only names some code object still uses: when it has
+//! doubled since it was last swept, the names no one else holds are
+//! dropped from it.
+
+use std::borrow::Borrow;
+use std::collections::HashSet;
+use std::hash::{BuildHasherDefault, Hash, Hasher};
+use std::sync::{Arc, Mutex};
+
+/// An identifier in a code object's name tables (see the module docs).
+/// It reads as a `str`.
+#[derive(Clone)]
+pub struct Name(Arc<str>);
+
+impl Name {
+    /// The pooled name equal to `s`.
+    pub fn new(s: &str) -> Name {
+        POOL.lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .intern(s)
+    }
+
+    #[inline]
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+
+    /// Whether `a` and `b` share one allocation (pooled names always do
+    /// when equal).
+    #[inline]
+    pub fn ptr_eq(a: &Name, b: &Name) -> bool {
+        Arc::ptr_eq(&a.0, &b.0)
+    }
+}
+
+/// The pool's hasher: FNV-1a, which is quick over short identifiers.
+struct Fnv(u64);
+
+impl Default for Fnv {
+    fn default() -> Self {
+        Fnv(0xcbf2_9ce4_8422_2325)
+    }
+}
+
+impl Hasher for Fnv {
+    fn finish(&self) -> u64 {
+        self.0
+    }
+
+    fn write(&mut self, bytes: &[u8]) {
+        let mut h = self.0;
+        for &b in bytes {
+            h = (h ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3);
+        }
+        self.0 = h;
+    }
+}
+
+struct Pool {
+    names: Option<HashSet<Arc<str>, BuildHasherDefault<Fnv>>>,
+    /// The size past which the next insertion sweeps unused names.
+    sweep_at: usize,
+}
+
+static POOL: Mutex<Pool> = Mutex::new(Pool {
+    names: None,
+    sweep_at: 4096,
+});
+
+impl Pool {
+    fn intern(&mut self, s: &str) -> Name {
+        let names = self.names.get_or_insert_with(HashSet::default);
+        if let Some(name) = names.get(s) {
+            return Name(name.clone());
+        }
+        if names.len() >= self.sweep_at {
+            names.retain(|name| Arc::strong_count(name) > 1);
+            self.sweep_at = (names.len() * 2).max(4096);
+        }
+        let name: Arc<str> = Arc::from(s);
+        names.insert(name.clone());
+        Name(name)
+    }
+}
+
+impl Default for Name {
+    fn default() -> Self {
+        Name::new("")
+    }
+}
+
+impl std::ops::Deref for Name {
+    type Target = str;
+
+    #[inline]
+    fn deref(&self) -> &str {
+        &self.0
+    }
+}
+
+impl AsRef<str> for Name {
+    fn as_ref(&self) -> &str {
+        &self.0
+    }
+}
+
+impl Borrow<str> for Name {
+    fn borrow(&self) -> &str {
+        &self.0
+    }
+}
+
+impl PartialEq for Name {
+    #[inline]
+    fn eq(&self, other: &Name) -> bool {
+        Arc::ptr_eq(&self.0, &other.0) || *self.0 == *other.0
+    }
+}
+
+impl Eq for Name {}
+
+impl PartialEq<str> for Name {
+    fn eq(&self, other: &str) -> bool {
+        &*self.0 == other
+    }
+}
+
+impl PartialEq<&str> for Name {
+    fn eq(&self, other: &&str) -> bool {
+        &*self.0 == *other
+    }
+}
+
+impl PartialEq<String> for Name {
+    fn eq(&self, other: &String) -> bool {
+        *self.0 == **other
+    }
+}
+
+impl PartialEq<Name> for str {
+    fn eq(&self, other: &Name) -> bool {
+        self == &*other.0
+    }
+}
+
+impl PartialEq<Name> for &str {
+    fn eq(&self, other: &Name) -> bool {
+        *self == &*other.0
+    }
+}
+
+impl PartialEq<Name> for String {
+    fn eq(&self, other: &Name) -> bool {
+        **self == *other.0
+    }
+}
+
+impl PartialOrd for Name {
+    fn partial_cmp(&self, other: &Name) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for Name {
+    fn cmp(&self, other: &Name) -> std::cmp::Ordering {
+        self.0.cmp(&other.0)
+    }
+}
+
+impl Hash for Name {
+    // As a `str`, which `Borrow<str>` requires.
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.0.hash(state);
+    }
+}
+
+impl std::fmt::Debug for Name {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        (*self.0).fmt(f)
+    }
+}
+
+impl std::fmt::Display for Name {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        (*self.0).fmt(f)
+    }
+}
+
+impl From<&str> for Name {
+    fn from(s: &str) -> Name {
+        Name::new(s)
+    }
+}
+
+impl From<String> for Name {
+    fn from(s: String) -> Name {
+        Name::new(&s)
+    }
+}
+
+impl From<&String> for Name {
+    fn from(s: &String) -> Name {
+        Name::new(s)
+    }
+}
+
+impl From<&Name> for Name {
+    fn from(s: &Name) -> Name {
+        s.clone()
+    }
+}
+
+impl From<Name> for String {
+    fn from(s: Name) -> String {
+        String::from(&*s.0)
+    }
+}
+
+impl From<&Name> for String {
+    fn from(s: &Name) -> String {
+        String::from(&*s.0)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn equal_names_share_one_allocation() {
+        let a = Name::new("pooled_name_a");
+        let b = Name::from(String::from("pooled_name_a"));
+        assert!(Name::ptr_eq(&a, &b));
+        assert_eq!(a, "pooled_name_a");
+        assert_eq!(a.as_str(), "pooled_name_a");
+        assert_ne!(a, Name::new("pooled_name_b"));
+    }
+
+    #[test]
+    fn sweeping_keeps_names_in_use() {
+        let kept = Name::new("kept_through_sweeps");
+        for i in 0..20_000 {
+            let _ = Name::new(&format!("transient_{i}"));
+        }
+        let again = Name::new("kept_through_sweeps");
+        assert!(Name::ptr_eq(&kept, &again));
+        let pool = POOL.lock().unwrap();
+        assert!(pool.names.as_ref().unwrap().len() < 20_000);
+    }
+}
diff --git a/crates/weavepy-compiler/src/native_code.rs b/crates/weavepy-compiler/src/native_code.rs
index b7e95b0a..5b839986 100644
--- a/crates/weavepy-compiler/src/native_code.rs
+++ b/crates/weavepy-compiler/src/native_code.rs
@@ -8,21 +8,23 @@
 //! mismatch or malformed input as a cache miss.
 //!
 //! Filenames aren't stored: every code object in a cached module shares
-//! the module's filename, which the reader supplies.
+//! the module's filename, which the reader supplies. A module's name
+//! tables refer to its distinct identifiers by number after their first
+//! appearance, so the reader decodes and pools each identifier once.
 
 use std::sync::Arc;
 
 use crate::bytecode::{CacheTable, Instruction, OpCode};
-use crate::{CodeObject, ColSpan, Constant, ExcHandler};
+use crate::{CodeObject, ColSpan, Constant, ExcHandler, Name};
 
 /// The layout revision; bump it whenever the encoding changes.
-pub const VERSION: u8 = 4;
+pub const VERSION: u8 = 5;
 
 /// Encode `code` (and its nested code objects). `None` for a code object
 /// the format doesn't carry: one with raw CPython wire overrides, or with
 /// a constant that has no fixed value.
 pub fn encode(code: &CodeObject) -> Option<Vec<u8>> {
-    let mut w = Writer(Vec::with_capacity(4096));
+    let mut w = Writer(Vec::with_capacity(4096), std::collections::HashMap::new());
     w.byte(VERSION);
     w.code(code)?;
     Some(w.0)
@@ -103,6 +105,7 @@ pub fn decode(bytes: &[u8], filename: &str) -> Option<CodeObject> {
         filename: Arc::from(filename),
         tables: Vec::new(),
         buffer: Arc::new(std::sync::OnceLock::new()),
+        names: Vec::new(),
     };
     let code = r.code(&mut module, 0)?;
     if r.pos != bytes.len() {
@@ -122,6 +125,9 @@ struct Module {
     tables: Vec<u8>,
     /// Where `tables` goes once the decoding is done.
     buffer: Arc<std::sync::OnceLock<Box<[u8]>>>,
+    /// The module's identifiers so far, in order of first appearance
+    /// (see [`Writer::names`]).
+    names: Vec<Name>,
 }
 
 impl Module {
@@ -134,7 +140,9 @@ impl Module {
     }
 }
 
-struct Writer(Vec<u8>);
+/// The output, and the number of each identifier written so far (see
+/// [`Writer::names`]).
+struct Writer(Vec<u8>, std::collections::HashMap<String, u32>);
 
 impl Writer {
     fn byte(&mut self, b: u8) {
@@ -165,6 +173,23 @@ impl Writer {
         }
     }
 
+    /// A name table: its length, then per name its number plus one when
+    /// the module wrote it before, else a zero and its text (which gives
+    /// it the next number).
+    fn names(&mut self, v: &[Name]) {
+        self.uint(v.len() as u64);
+        for s in v {
+            if let Some(&i) = self.1.get(s.as_str()) {
+                self.uint(u64::from(i) + 1);
+            } else {
+                let i = self.1.len() as u32;
+                self.1.insert(s.as_str().to_owned(), i);
+                self.uint(0);
+                self.bytes(s.as_bytes());
+            }
+        }
+    }
+
     fn u32s(&mut self, v: &[u32]) {
         self.uint(v.len() as u64);
         for &x in v {
@@ -234,10 +259,10 @@ impl Writer {
         for k in &c.constants {
             self.constant(k)?;
         }
-        self.strs(&c.names);
-        self.strs(&c.varnames);
-        self.strs(&c.freevars);
-        self.strs(&c.cellvars);
+        self.names(&c.names);
+        self.names(&c.varnames);
+        self.names(&c.freevars);
+        self.names(&c.cellvars);
         self.uint(c.exception_table.len() as u64);
         for h in &c.exception_table {
             self.uint(u64::from(h.start));
@@ -250,7 +275,7 @@ impl Writer {
         self.line_runs(&c.linetable);
         // The column spans as one length-prefixed block, which a reader
         // keeps encoded until something reads a column (see `ColTable`).
-        let mut cols = Writer(Vec::new());
+        let mut cols = Writer(Vec::new(), std::collections::HashMap::new());
         cols.col_runs(&c.coltable)?;
         self.bytes(&cols.0);
         self.uint(u64::from(c.arg_count));
@@ -428,6 +453,24 @@ impl Reader<'_> {
         Some(v)
     }
 
+    /// A name table [`Writer::names`] wrote.
+    fn names(&mut self, module: &mut Module) -> Option<Vec<Name>> {
+        let n = self.len()?;
+        let mut v = Vec::with_capacity(n);
+        for _ in 0..n {
+            let name = match self.uint()? {
+                0 => {
+                    let name = Name::new(std::str::from_utf8(self.raw()?).ok()?);
+                    module.names.push(name.clone());
+                    name
+                }
+                i => module.names.get(usize::try_from(i - 1).ok()?)?.clone(),
+            };
+            v.push(name);
+        }
+        Some(v)
+    }
+
     fn u32s(&mut self) -> Option<Vec<u32>> {
         let n = self.len()?;
         let mut v = Vec::with_capacity(n);
@@ -504,10 +547,10 @@ impl Reader<'_> {
         for _ in 0..n {
             constants.push(self.constant(module, depth)?);
         }
-        let names = self.strs()?;
-        let varnames = self.strs()?;
-        let freevars = self.strs()?;
-        let cellvars = self.strs()?;
+        let names = self.names(module)?;
+        let varnames = self.names(module)?;
+        let freevars = self.names(module)?;
+        let cellvars = self.names(module)?;
         let n = self.len()?;
         let mut exception_table = Vec::with_capacity(n);
         for _ in 0..n {
@@ -627,7 +670,7 @@ mod tests {
                 Instruction::new(OpCode::ReturnValue, 0),
             ],
             constants: vec![Constant::Int(-5)],
-            varnames: vec!["x".to_owned()],
+            varnames: vec!["x".into()],
             linetable: vec![1, 2, 2].into(),
             coltable: vec![ColSpan::default(); 3].into(),
             arg_count: 1,
@@ -660,7 +703,7 @@ mod tests {
                 ))),
                 Constant::Code(Arc::new(inner)),
             ],
-            names: vec!["print".to_owned()],
+            names: vec!["print".into()],
             exception_table: vec![ExcHandler {
                 start: 0,
                 end: 1,
diff --git a/crates/weavepy-jit/src/analyze.rs b/crates/weavepy-jit/src/analyze.rs
index 5421532e..6c42c23b 100644
--- a/crates/weavepy-jit/src/analyze.rs
+++ b/crates/weavepy-jit/src/analyze.rs
@@ -2010,7 +2010,7 @@ fn plan_rewrite(
                 && ins[i - 2].arg == 0
                 && matches!(ins[i - 3].op, OpCode::LoadMethodAttr)
                 && matches!(ins[i - 4].op, OpCode::LoadFast)
-                && code.names.get(ins[i - 3].arg as usize).map(String::as_str) == Some("items")
+                && code.names.get(ins[i - 3].arg as usize).map(|n| n.as_str()) == Some("items")
             {
                 plan.pair_src
                     .insert(i - 1, PairSrc::DictItems(ins[i - 4].arg));
@@ -2045,7 +2045,7 @@ fn plan_rewrite(
         // backwards: GET_ITER, CALL k, k simple args, PUSH_NULL,
         // LOAD_GLOBAL <range>.
         if let Some((callee, push_null, step_nop, pops)) = range_prefix(code, &plan, &targets, i) {
-            let name = code.names[ins[callee].arg as usize].clone();
+            let name = code.names[ins[callee].arg as usize].to_string();
             if !plan.guards.iter().any(|g| g.name == name) {
                 plan.guards.push(GlobalGuard {
                     name,
@@ -2352,7 +2352,7 @@ fn plan_rewrite(
     for (i, item) in ins.iter().enumerate() {
         if matches!(item.op, OpCode::LoadGlobal)
             && matches!(
-                code.names.get(item.arg as usize).map(String::as_str),
+                code.names.get(item.arg as usize).map(|n| n.as_str()),
                 Some("eval" | "exec" | "locals" | "vars" | "dir" | "super" | "breakpoint")
             )
         {
@@ -2482,7 +2482,7 @@ fn plan_rewrite(
             let name = &code.names[ins[pc].arg as usize];
             if !plan.guards.iter().any(|g| g.name == *name) {
                 plan.guards.push(GlobalGuard {
-                    name: name.clone(),
+                    name: name.to_string(),
                     expect,
                 });
             }
@@ -2516,7 +2516,7 @@ fn plan_rewrite(
                 plan.obj_globals.insert(item.arg, (token, lane));
                 if !plan.guards.iter().any(|g| g.name == *name) {
                     plan.guards.push(GlobalGuard {
-                        name: name.clone(),
+                        name: name.to_string(),
                         expect: ResolvedGlobal::ObjGlobal { token, lane },
                     });
                 }
@@ -2531,7 +2531,7 @@ fn plan_rewrite(
                 let name = &code.names[item.arg as usize];
                 if !plan.guards.iter().any(|g| g.name == *name) {
                     plan.guards.push(GlobalGuard {
-                        name: name.clone(),
+                        name: name.to_string(),
                         expect: resolved,
                     });
                 }
@@ -2545,7 +2545,7 @@ fn plan_rewrite(
                 plan.obj_globals.insert(item.arg, (token, lane));
                 if !plan.guards.iter().any(|g| g.name == *name) {
                     plan.guards.push(GlobalGuard {
-                        name: name.clone(),
+                        name: name.to_string(),
                         expect: ResolvedGlobal::ObjGlobal { token, lane },
                     });
                 }
@@ -3494,7 +3494,7 @@ impl CtorState {
             CtorFieldSrc::Param(j) => arg_lanes.get(j as usize).copied()?,
             CtorFieldSrc::Lane(l) => l,
         };
-        Some((cls.clone(), field_idx, lane))
+        Some((cls.to_string(), field_idx, lane))
     }
 
     /// Record one same-body new-key attribute store on a direct local
diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index b15b93b5..574692a1 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -4573,10 +4573,10 @@ fn code_replace(args: &[Object], kwargs: &[(String, Object)]) -> Result<Object,
             "co_posonlyargcount" => nc.posonly_count = want_u32(v, "co_posonlyargcount")?,
             "co_kwonlyargcount" => nc.kwonly_count = want_u32(v, "co_kwonlyargcount")?,
             "co_nlocals" => requested_nlocals = Some(want_u32(v, "co_nlocals")?),
-            "co_varnames" => nc.varnames = want_str_seq(v, "co_varnames")?,
-            "co_names" => nc.names = want_str_seq(v, "co_names")?,
-            "co_freevars" => nc.freevars = want_str_seq(v, "co_freevars")?,
-            "co_cellvars" => nc.cellvars = want_str_seq(v, "co_cellvars")?,
+            "co_varnames" => nc.varnames = names_of(want_str_seq(v, "co_varnames")?),
+            "co_names" => nc.names = names_of(want_str_seq(v, "co_names")?),
+            "co_freevars" => nc.freevars = names_of(want_str_seq(v, "co_freevars")?),
+            "co_cellvars" => nc.cellvars = names_of(want_str_seq(v, "co_cellvars")?),
             "co_stacksize" => {
                 nc.wire.get_or_insert_with(Default::default).stacksize =
                     Some(want_u32(v, "co_stacksize")?);
@@ -4727,7 +4727,7 @@ pub fn foreign_code_object(
         name,
         qualname,
         filename: filename.into(),
-        varnames,
+        varnames: names_of(varnames),
         arg_count,
         posonly_count,
         kwonly_count,
@@ -4749,6 +4749,11 @@ pub fn foreign_code_object(
     Object::Code(Rc::new(nc))
 }
 
+/// A code object's name table from a constructor or `replace()` argument.
+fn names_of(names: Vec<String>) -> Vec<weavepy_compiler::Name> {
+    names.iter().map(weavepy_compiler::Name::from).collect()
+}
+
 /// Pin raw CPython `co_code` bytes on `nc` (RFC 0060 — `CodeType(...)` /
 /// `replace(co_code=…)`). When the stream decodes back into WeavePy
 /// instructions the code object stays executable; otherwise executing it
@@ -4922,10 +4927,10 @@ pub(crate) fn code_type_call(
         qualname,
         filename: filename.into(),
         constants,
-        names,
-        varnames,
-        freevars,
-        cellvars,
+        names: names_of(names),
+        varnames: names_of(varnames),
+        freevars: names_of(freevars),
+        cellvars: names_of(cellvars),
         arg_count,
         posonly_count,
         kwonly_count,
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index b86f5233..60c04817 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -7976,7 +7976,7 @@ impl Interpreter {
                                 .code
                                 .names
                                 .get(arg as usize)
-                                .map_or("?", String::as_str),
+                                .map_or("?", |n| n.as_str()),
                             recv,
                         );
                     }
@@ -11292,7 +11292,7 @@ impl Interpreter {
             || call.op != OpCode::Call
             || call.arg != 1
             || frame.builtins_obj().is_some()
-            || frame.code.names.get(name_idx as usize).map(String::as_str) != Some("len")
+            || frame.code.names.get(name_idx as usize).map(|n| n.as_str()) != Some("len")
         {
             return None;
         }
@@ -22715,7 +22715,7 @@ impl Interpreter {
             return None;
         };
         if Rc::as_ptr(f) as usize != self.leaf_fns().len_ptr
-            || code.names.get(name_idx as usize).map(String::as_str) != Some("len")
+            || code.names.get(name_idx as usize).map(|n| n.as_str()) != Some("len")
         {
             return None;
         }
@@ -32434,7 +32434,7 @@ impl Interpreter {
     fn name_at(&self, code: &CodeObject, arg: u32) -> Result<String, RuntimeError> {
         code.names
             .get(arg as usize)
-            .cloned()
+            .map(|n| n.to_string())
             .ok_or_else(|| RuntimeError::Internal("bad name index".to_owned()))
     }
 
@@ -43559,7 +43559,7 @@ impl Interpreter {
         };
         {
             let dict = d.borrow();
-            let cell_names: Vec<&String> = frame
+            let cell_names: Vec<&weavepy_compiler::Name> = frame
                 .code
                 .cellvars
                 .iter()
@@ -46275,7 +46275,7 @@ impl Interpreter {
                     .code
                     .names
                     .get(name_idx as usize)
-                    .map(String::as_str)
+                    .map(|n| n.as_str())
                     .ok_or_else(|| RuntimeError::Internal("bad name index".to_owned()))?;
                 if frame
                     .globals
@@ -46624,7 +46624,7 @@ impl Interpreter {
                         // genuine `__slots__` member descriptor for this
                         // name) and the version guard covers class-dict /
                         // MRO changes since.
-                        let name = frame.code.names.get(name_idx as usize).map(String::as_str);
+                        let name = frame.code.names.get(name_idx as usize).map(|n| n.as_str());
                         if let Some(name) = name {
                             let hit = {
                                 let slots = inst.slots.borrow();
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index c3295178..0901b6c0 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -722,7 +722,7 @@ impl PyFrame {
         let mirror = self.locals_mirror.borrow().clone()?;
         let snapshot = mirror.borrow();
         let varnames = &self.code.varnames;
-        let cell_names: Vec<&String> = self
+        let cell_names: Vec<&weavepy_compiler::Name> = self
             .code
             .cellvars
             .iter()
@@ -798,7 +798,7 @@ impl PyFrame {
             if name.starts_with('.') || out.iter().any(|n| n == name) {
                 continue;
             }
-            out.push(name.clone());
+            out.push(name.to_string());
         }
         out
     }
diff --git a/crates/weavepy-vm/src/specialize.rs b/crates/weavepy-vm/src/specialize.rs
index 6ab2cee4..ee6ebbc6 100644
--- a/crates/weavepy-vm/src/specialize.rs
+++ b/crates/weavepy-vm/src/specialize.rs
@@ -599,7 +599,7 @@ pub fn kw_uncovered_have_defaults(
         covered & (1 << slot) != 0
             || f.kw_defaults
                 .iter()
-                .any(|(n, _)| Some(n.as_str()) == code.varnames.get(slot).map(String::as_str))
+                .any(|(n, _)| Some(n.as_str()) == code.varnames.get(slot).map(|n| n.as_str()))
     })
 }
 
diff --git a/crates/weavepy-vm/src/stdlib/elementtree_native.rs b/crates/weavepy-vm/src/stdlib/elementtree_native.rs
index 878ca4f4..67c0f659 100644
--- a/crates/weavepy-vm/src/stdlib/elementtree_native.rs
+++ b/crates/weavepy-vm/src/stdlib/elementtree_native.rs
@@ -1630,7 +1630,7 @@ const EP_ITERFIND: usize = 3;
 /// The closure `f` holds for free variable `name`, when it is a `str`.
 fn closure_str(
     f: &crate::object::PyFunction,
-    freevars: &[String],
+    freevars: &[weavepy_compiler::Name],
     name: &str,
 ) -> Option<SharedStr> {
     let i = freevars.iter().position(|v| v == name)?;
diff --git a/crates/weavepy-vm/src/stdlib/marshal_mod.rs b/crates/weavepy-vm/src/stdlib/marshal_mod.rs
index c0215949..1f740652 100644
--- a/crates/weavepy-vm/src/stdlib/marshal_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/marshal_mod.rs
@@ -1184,8 +1184,13 @@ impl Scan {
         // Names never pass through the compiler's constant cache, so an
         // equal string *constant* keeps its own object: the canonical
         // constant is not decided here, only the identifier set.
-        for name in co.names.iter().chain(cp.localsplusnames.iter()) {
-            self.identifiers.insert(name.clone());
+        for name in co
+            .names
+            .iter()
+            .map(|n| n.as_str())
+            .chain(cp.localsplusnames.iter().map(String::as_str))
+        {
+            self.identifiers.insert(name.to_owned());
         }
         self.identifiers.insert(co.name.clone());
         self.qualnames_done.insert(co.qualname.clone());
@@ -1258,8 +1263,8 @@ fn bigint_to_15bit(b: &BigInt) -> (i32, Vec<u16>) {
 }
 
 /// Build a `marshal` tuple of interned-string objects.
-fn strs_to_tuple(items: &[String]) -> Object {
-    Object::new_tuple(items.iter().map(|s| Object::from_str(s.clone())).collect())
+fn strs_to_tuple<S: AsRef<str>>(items: &[S]) -> Object {
+    Object::new_tuple(items.iter().map(|s| Object::from_str(s.as_ref())).collect())
 }
 
 // ---------- reader ----------
@@ -1741,10 +1746,25 @@ impl<'a> MarshalReader<'a> {
             jit_hint: weavepy_compiler::JitHint::default(),
             instructions: decoded.instructions,
             constants,
-            names: tuple_of_strings(&names, "co_names")?,
-            varnames: decoded.varnames,
-            freevars: decoded.freevars,
-            cellvars: decoded.cellvars,
+            names: tuple_of_strings(&names, "co_names")?
+                .iter()
+                .map(weavepy_compiler::Name::from)
+                .collect(),
+            varnames: decoded
+                .varnames
+                .iter()
+                .map(weavepy_compiler::Name::from)
+                .collect(),
+            freevars: decoded
+                .freevars
+                .iter()
+                .map(weavepy_compiler::Name::from)
+                .collect(),
+            cellvars: decoded
+                .cellvars
+                .iter()
+                .map(weavepy_compiler::Name::from)
+                .collect(),
             exception_table: decoded.exception_table,
             linetable: decoded.linetable.into(),
             // PEP-657 columns recovered from long-form location entries
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index c2feeca4..9494ef9d 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -2288,7 +2288,7 @@ fn ctor_field_plan(icode: &CodeObject) -> Option<Vec<(String, CtorFieldSrc)>> {
                 if recv.op != OpCode::LoadFast || recv.arg != 0 || store.op != OpCode::StoreAttr {
                     return None;
                 }
-                let name = icode.names.get(store.arg as usize)?.clone();
+                let name = icode.names.get(store.arg as usize)?.to_string();
                 match fields.iter_mut().find(|(n, _)| *n == name) {
                     Some(slot) => slot.1 = src,
                     None => fields.push((name, src)),
@@ -3150,12 +3150,13 @@ fn attr_chain_step(obj: &Object, name: &str) -> Option<Object> {
 /// residue for locals bound from a live list's elements: the probe
 /// predicts from a representative, and the burned fingerprints
 /// re-validate per access).
-fn walk_attr_path(frame: &super::Frame, slot: u32, path: &[String]) -> Option<Object> {
+fn walk_attr_path<S: AsRef<str>>(frame: &super::Frame, slot: u32, path: &[S]) -> Option<Object> {
     let mut cur = {
         let locals = frame.locals.borrow();
         locals.get(slot as usize)?.clone()
     };
     for name in path {
+        let name = name.as_ref();
         if name == weavepy_jit::ELEM_SENTINEL {
             cur = exemplar_element(&cur)?;
             continue;
@@ -4014,7 +4015,7 @@ fn inline_fields_of(recv: &Object, code: &CodeObject, ver: u64) -> Vec<InlineFie
             _ => continue,
         };
         out.push(InlineField {
-            name: name.clone(),
+            name: name.to_string(),
             lane,
             at,
         });
@@ -5851,7 +5852,7 @@ unsafe fn call_py_with_gaps(
         if kwargs.is_empty() && args.len() == j {
             args.push(v);
         } else if let Some(name) = code.varnames.get(j) {
-            kwargs.push((name.clone(), v));
+            kwargs.push((name.to_string(), v));
         }
     }
     let res = call_with_activation_shell(interp, ctx, jf, |i| {

From 285c9054428d6b97bdf14bccb45a350e86528b6a Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 09:31:26 -0700
Subject: [PATCH 068/173] perf: start each class's leaf attribute cache at
 eight entries

A class's leaf attribute cache allocated all 32 of its 40-byte entries
on its first fill, 1.3 KB per class that answered any name, though most
classes answer only a few (importing asyncio fills about 250 of them).

The table now starts at eight entries and grows to 32 only when a name
would evict another current one from both of its slots, moving the
entries it holds. The table stays one (tagged) pointer, and the lookup
masks the slot index by the table's size.

Peak RSS: import asyncio 27.71 -> 27.48 MB, import typing, dataclasses,
enum 21.57 -> 21.39 MB, -c pass 11.02 -> 10.87 MB.
---
 crates/weavepy-vm/src/types.rs | 189 ++++++++++++++++++++++++++++-----
 1 file changed, 165 insertions(+), 24 deletions(-)

diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index c834ba11..3cadb8a9 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -487,12 +487,24 @@ pub(crate) mod type_cache {
 /// address and the class's attribute version. A small direct-mapped
 /// cache per class: polymorphic sites (one receiver class per iteration)
 /// find every class's answer here with no site cache at all. Storage is
-/// allocated on the first fill; unused classes occupy only one pointer.
+/// allocated on the first fill, [`LEAF_ATTRS_SMALL`] entries at first and
+/// [`LEAF_ATTRS_LARGE`] once two names compete for a full slot pair (most
+/// classes answer only a few names); unused classes occupy only one
+/// pointer.
 ///
 /// Read and written only from the dispatch loop with the GIL held (the
 /// burst is off in free-threaded mode), like the dispatch loop's other
-/// side caches.
-pub struct LeafAttrCache(std::cell::UnsafeCell<Option<Box<[LeafAttrEntry; 32]>>>);
+/// side caches. A reference [`LeafAttrCache::get`] returns must not be
+/// held across a [`LeafAttrCache::set`], which may move the entries.
+pub struct LeafAttrCache(std::cell::Cell<usize>);
+
+/// The entries a class's first [`LeafAttrCache`] fill allocates.
+const LEAF_ATTRS_SMALL: usize = 8;
+/// The entries a [`LeafAttrCache`] grows to.
+const LEAF_ATTRS_LARGE: usize = 32;
+/// The tag bit of a [`LeafAttrCache`]'s table address that marks a
+/// table of [`LEAF_ATTRS_SMALL`] entries (entries are word-aligned).
+const LEAF_ATTRS_SMALL_TAG: usize = 1;
 
 // SAFETY: see the type docs — GIL-serialized access from one thread at a
 // time, never across a Python call.
@@ -557,7 +569,86 @@ pub enum LeafAttrKind {
 
 impl LeafAttrCache {
     pub fn new() -> Self {
-        Self(std::cell::UnsafeCell::new(None))
+        Self(std::cell::Cell::new(0))
+    }
+
+    /// The table, if one was allocated.
+    #[inline]
+    fn table(&self) -> Option<&[LeafAttrEntry]> {
+        let t = self.0.get();
+        if t == 0 {
+            return None;
+        }
+        let len = if t & LEAF_ATTRS_SMALL_TAG != 0 {
+            LEAF_ATTRS_SMALL
+        } else {
+            LEAF_ATTRS_LARGE
+        };
+        // SAFETY: a nonzero address (tag removed) is a live table of `len`
+        // entries; GIL-serialized, and no `&mut` escapes `set`.
+        Some(unsafe {
+            std::slice::from_raw_parts((t & !LEAF_ATTRS_SMALL_TAG) as *const LeafAttrEntry, len)
+        })
+    }
+
+    /// The table for writing, allocated (small) on first use.
+    #[allow(clippy::mut_from_ref)]
+    fn table_mut(&self) -> &mut [LeafAttrEntry] {
+        if self.0.get() == 0 {
+            let t: Box<[LeafAttrEntry; LEAF_ATTRS_SMALL]> =
+                Box::new(std::array::from_fn(|_| LeafAttrEntry::EMPTY));
+            self.0
+                .set(Box::into_raw(t) as usize | LEAF_ATTRS_SMALL_TAG);
+        }
+        let t = self.0.get();
+        let len = if t & LEAF_ATTRS_SMALL_TAG != 0 {
+            LEAF_ATTRS_SMALL
+        } else {
+            LEAF_ATTRS_LARGE
+        };
+        // SAFETY: as `table`; the exclusive reference lives only for the
+        // caller's update.
+        unsafe {
+            std::slice::from_raw_parts_mut((t & !LEAF_ATTRS_SMALL_TAG) as *mut LeafAttrEntry, len)
+        }
+    }
+
+    /// Release the table, leaving none.
+    fn free(&self) {
+        let t = self.0.replace(0);
+        if t == 0 {
+            return;
+        }
+        let p = t & !LEAF_ATTRS_SMALL_TAG;
+        // SAFETY: the table was allocated as this boxed array (see
+        // `table_mut` and `grow`) and is no longer reachable.
+        unsafe {
+            if t & LEAF_ATTRS_SMALL_TAG != 0 {
+                drop(Box::from_raw(p as *mut [LeafAttrEntry; LEAF_ATTRS_SMALL]));
+            } else {
+                drop(Box::from_raw(p as *mut [LeafAttrEntry; LEAF_ATTRS_LARGE]));
+            }
+        }
+    }
+
+    /// Move a small table's entries into a large one.
+    #[cold]
+    fn grow(&self) {
+        let mut large: Box<[LeafAttrEntry; LEAF_ATTRS_LARGE]> =
+            Box::new(std::array::from_fn(|_| LeafAttrEntry::EMPTY));
+        for e in self.table_mut() {
+            let e = std::mem::replace(e, LeafAttrEntry::EMPTY);
+            if e.name == 0 {
+                continue;
+            }
+            let (i, j) = (Self::index(e.name), Self::alt(e.name));
+            let k = if large[i].name == 0 { i } else { j };
+            if large[k].name == 0 {
+                large[k] = e;
+            }
+        }
+        self.free();
+        self.0.set(Box::into_raw(large) as usize);
     }
 
     #[inline]
@@ -576,37 +667,51 @@ impl LeafAttrCache {
     /// The cached kind for `name` under `ver`, if present.
     #[inline]
     pub fn get(&self, name: usize, ver: u64) -> Option<&LeafAttrKind> {
-        // SAFETY: GIL-serialized; no `&mut` escapes `set`.
-        let t = unsafe { &*self.0.get() }.as_deref()?;
-        let e = &t[Self::index(name)];
+        let t = self.table()?;
+        // (Both table lengths are powers of two.)
+        let mask = t.len() - 1;
+        let e = &t[Self::index(name) & mask];
         if e.name == name && e.ver == ver {
             return Some(&e.kind);
         }
-        let e = &t[Self::alt(name)];
+        let e = &t[Self::alt(name) & mask];
         (e.name == name && e.ver == ver).then_some(&e.kind)
     }
 
     #[inline]
     pub fn set(&self, name: usize, ver: u64, kind: LeafAttrKind) {
-        // SAFETY: GIL-serialized; the exclusive reference lives only for
-        // the assignment.
-        let t = unsafe { &mut *self.0.get() }.get_or_insert_with(|| {
-            Box::new(std::array::from_fn(|_| LeafAttrEntry {
-                name: 0,
-                ver: 0,
-                kind: LeafAttrKind::Other,
-            }))
-        });
-        let (i, j) = (Self::index(name), Self::alt(name));
         // The first slot unless it holds another name that is still
         // current; then the second, unless that one is too (evict the
-        // first).
+        // first, or grow a small table instead).
         let busy = |e: &LeafAttrEntry| e.name != 0 && e.name != name && e.ver == ver;
+        let mut t = self.table_mut();
+        let mut mask = t.len() - 1;
+        let (i, j) = (Self::index(name), Self::alt(name));
+        if t.len() == LEAF_ATTRS_SMALL && busy(&t[i & mask]) && busy(&t[j & mask]) {
+            self.grow();
+            t = self.table_mut();
+            mask = t.len() - 1;
+        }
+        let (i, j) = (i & mask, j & mask);
         let k = if !busy(&t[i]) || busy(&t[j]) { i } else { j };
         t[k] = LeafAttrEntry { name, ver, kind };
     }
 }
 
+impl LeafAttrEntry {
+    const EMPTY: LeafAttrEntry = LeafAttrEntry {
+        name: 0,
+        ver: 0,
+        kind: LeafAttrKind::Other,
+    };
+}
+
+impl Drop for LeafAttrCache {
+    fn drop(&mut self) {
+        self.free();
+    }
+}
+
 impl Default for LeafAttrCache {
     fn default() -> Self {
         Self::new()
@@ -627,22 +732,58 @@ impl std::fmt::Debug for LeafAttrCache {
 
 #[cfg(test)]
 mod leaf_attr_cache_tests {
-    use super::{LeafAttrCache, LeafAttrKind};
+    use super::{LeafAttrCache, LeafAttrKind, LEAF_ATTRS_LARGE, LEAF_ATTRS_SMALL};
     use crate::object::Object;
     use crate::shared_value::SharedStr;
 
+    #[test]
+    fn small_table_grows_when_names_compete_and_keeps_values() {
+        let cache = LeafAttrCache::new();
+        let text = SharedStr::from("moved value");
+        let first = 16;
+        cache.set(first, 1, LeafAttrKind::Value(Object::Str(text.clone())));
+        assert_eq!(cache.table().map(<[_]>::len), Some(LEAF_ATTRS_SMALL));
+        // Two names sharing `first`'s slot pair in the small table, but
+        // neither of its slot in the large one.
+        let small = |n: usize| {
+            (
+                LeafAttrCache::index(n) % LEAF_ATTRS_SMALL,
+                LeafAttrCache::alt(n) % LEAF_ATTRS_SMALL,
+            )
+        };
+        let home = LeafAttrCache::index(first);
+        let rivals: Vec<usize> = (17..1_000_000)
+            .filter(|&n| {
+                small(n) == small(first)
+                    && LeafAttrCache::index(n) != home
+                    && LeafAttrCache::alt(n) != home
+            })
+            .take(2)
+            .collect();
+        cache.set(rivals[0], 1, LeafAttrKind::InstanceOnly);
+        assert_eq!(cache.table().map(<[_]>::len), Some(LEAF_ATTRS_SMALL));
+        cache.set(rivals[1], 1, LeafAttrKind::InstanceOnly);
+        assert_eq!(cache.table().map(<[_]>::len), Some(LEAF_ATTRS_LARGE));
+        assert!(matches!(
+            cache.get(first, 1),
+            Some(LeafAttrKind::Value(Object::Str(value))) if SharedStr::ptr_eq(value, &text)
+        ));
+        assert!(matches!(cache.get(rivals[1], 1), Some(LeafAttrKind::InstanceOnly)));
+        assert_eq!(SharedStr::strong_count(&text), 2);
+        drop(cache);
+        assert_eq!(SharedStr::strong_count(&text), 1);
+    }
+
     #[test]
     fn cold_cache_and_its_clone_do_not_allocate_entries() {
         let cache = LeafAttrCache::new();
         assert_eq!(std::mem::size_of_val(&cache), std::mem::size_of::<usize>());
         assert!(cache.get(16, 1).is_none());
-        // SAFETY: this test owns the cache exclusively.
-        assert!(unsafe { &*cache.0.get() }.is_none());
+        assert!(cache.table().is_none());
         cache.set(16, 1, LeafAttrKind::InstanceOnly);
         let other = cache.clone();
         assert!(other.get(16, 1).is_none());
-        // SAFETY: this test owns the clone exclusively.
-        assert!(unsafe { &*other.0.get() }.is_none());
+        assert!(other.table().is_none());
     }
 
     #[test]

From 6d834ff01ba913f5e90119419a2fd25a137047a4 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 10:02:36 -0700
Subject: [PATCH 069/173] perf: keep tier 2's analysis and code generator out
 of imports

Three changes to when tier 2 spends time and memory:

- The engine is built once an analysis admits some code, not before the
  first analysis: code the analysis rejects (every attempt in sudoku)
  never pays for building one (`JitEngine::compile_frame_lazily`).
- The code generator's background warm-up starts halfway to the work
  the current phase admits a compile at. During start-up and imports
  that is 64 times the threshold, so a warm-up at half the threshold
  paged in the code generator for compiles that never came.
- During start-up and imports a body longer than 512 instructions needs
  work in proportion to its length before it is analyzed. Analysis
  costs about the body's length: `import statistics` spent 12 ms and a
  sixth of its instructions analyzing `re._parser._parse` twice only to
  reject it. Such a body can still compile at the ordinary threshold
  once the imports are done.

import pathlib, datetime, urllib.parse, statistics: 325.2M -> 267.6M
instructions, 23.97 -> 23.38 MB peak RSS. The other import probes and
the suite's peak RSS don't move.
---
 crates/weavepy-jit/src/engine.rs | 62 ++++++++++++++++++++++++++------
 crates/weavepy-jit/src/lib.rs    |  1 +
 crates/weavepy-vm/src/tier2.rs   | 56 +++++++++++++++++++++++------
 crates/weavepy-vm/src/types.rs   |  8 +++--
 4 files changed, 103 insertions(+), 24 deletions(-)

diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs
index 634c4d7f..48e4dd25 100644
--- a/crates/weavepy-jit/src/engine.rs
+++ b/crates/weavepy-jit/src/engine.rs
@@ -666,19 +666,36 @@ impl JitEngine {
         inline_method: &mut dyn FnMut(u32, &[JitType], u32) -> Option<InlineMethod>,
         slot_member: &mut dyn FnMut(&AttrSiteMeta) -> bool,
     ) -> Result<CompiledFrame, JitVerdict> {
-        let mut tfunc = crate::analyze::analyze_frame(code, resolve, probes)?;
-        if calls_dynamically(&tfunc) {
-            return Err(JitVerdict::UnsupportedOpcode("dynamic call (loop-free)"));
-        }
-        if mostly_generic(&tfunc) {
-            return Err(JitVerdict::UnsupportedOpcode("mostly generic operations"));
-        }
-        for site in &mut tfunc.attr_sites {
-            site.slot_member = slot_member(site);
-        }
+        let tfunc = analyze_admitted(code, resolve, probes, slot_member)?;
         self.compile_tfunc_methods(&tfunc, direct, direct_method, inline_method)
     }
 
+    /// [`Self::compile_frame_direct`] on the engine in `engine`, built
+    /// (see [`Self::new`]) only once the analysis admits the code: code
+    /// the analysis rejects never pays for building one. When the host
+    /// can't have one, `engine` stays empty and the verdict is
+    /// [`NO_HOST_ENGINE`].
+    #[allow(clippy::too_many_arguments)]
+    pub fn compile_frame_lazily(
+        engine: &mut Option<JitEngine>,
+        code: &CodeObject,
+        resolve: &mut dyn FnMut(&str) -> ResolvedGlobal,
+        probes: &mut Probes<'_>,
+        direct: &mut dyn FnMut(u32) -> Option<DirectLeaf>,
+        direct_method: &mut dyn FnMut(u32) -> Option<DirectLeaf>,
+        inline_method: &mut dyn FnMut(u32, &[JitType], u32) -> Option<InlineMethod>,
+        slot_member: &mut dyn FnMut(&AttrSiteMeta) -> bool,
+    ) -> Result<CompiledFrame, JitVerdict> {
+        let tfunc = analyze_admitted(code, resolve, probes, slot_member)?;
+        if engine.is_none() {
+            *engine = JitEngine::new();
+        }
+        let Some(engine) = engine.as_mut() else {
+            return Err(JitVerdict::UnsupportedOpcode(NO_HOST_ENGINE));
+        };
+        engine.compile_tfunc_methods(&tfunc, direct, direct_method, inline_method)
+    }
+
     /// Compile an already-analyzed [`TFunc`] (also the unit-test entry).
     pub fn compile_tfunc(&mut self, tfunc: &TFunc) -> Result<CompiledFrame, JitVerdict> {
         self.compile_tfunc_direct(tfunc, &mut |_| None)
@@ -1146,6 +1163,31 @@ fn calls_dynamically(tfunc: &TFunc) -> bool {
 /// through a helper that redoes the lookups the interpreter's caches
 /// remember, while each typed one saves only a dispatch. A loop-free
 /// body is left alone: other compiled code calls it directly.
+/// The [`JitEngine::compile_frame_lazily`] verdict for a host the code
+/// generator can't target.
+pub const NO_HOST_ENGINE: &str = "no code generator for the host";
+
+/// Analyze `code` for [`JitEngine::compile_frame_direct`] and apply the
+/// admission rules that need no code generator.
+fn analyze_admitted(
+    code: &CodeObject,
+    resolve: &mut dyn FnMut(&str) -> ResolvedGlobal,
+    probes: &mut Probes<'_>,
+    slot_member: &mut dyn FnMut(&AttrSiteMeta) -> bool,
+) -> Result<TFunc, JitVerdict> {
+    let mut tfunc = crate::analyze::analyze_frame(code, resolve, probes)?;
+    if calls_dynamically(&tfunc) {
+        return Err(JitVerdict::UnsupportedOpcode("dynamic call (loop-free)"));
+    }
+    if mostly_generic(&tfunc) {
+        return Err(JitVerdict::UnsupportedOpcode("mostly generic operations"));
+    }
+    for site in &mut tfunc.attr_sites {
+        site.slot_member = slot_member(site);
+    }
+    Ok(tfunc)
+}
+
 fn mostly_generic(tfunc: &TFunc) -> bool {
     let Some((percent, loops_only)) = generic_share_limit() else {
         return false;
diff --git a/crates/weavepy-jit/src/lib.rs b/crates/weavepy-jit/src/lib.rs
index e42d5f7c..ba2190d1 100644
--- a/crates/weavepy-jit/src/lib.rs
+++ b/crates/weavepy-jit/src/lib.rs
@@ -36,6 +36,7 @@ pub use analyze::{
 };
 pub use engine::{
     admit_generic_loops, CompiledFrame, DirectLeaf, FieldAt, InlineMethod, JitEngine, OpMix,
+    NO_HOST_ENGINE,
 };
 pub use ir::{
     ArithKind, AttrSiteMeta, BlockId, CalleeSpanMeta, CmpKind, CompSavedMeta, CtorFieldSrc,
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index 9494ef9d..20608e83 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -1098,6 +1098,29 @@ fn compile_allowed(counter: u32, threshold: u32) -> bool {
 /// attempt costs milliseconds.
 const IMPORT_THRESHOLD_FACTOR: u32 = 64;
 
+/// Bodies longer than this many instructions need proportionally more
+/// work before start-up or an import analyzes them (see
+/// [`import_size_budget`]).
+const IMPORT_LARGE_BODY: usize = 512;
+
+/// The counter a code object needs before tier 2 analyzes it, beyond
+/// [`compile_allowed`]'s, when this phase admits compiles only at
+/// `admit_at` (more than the threshold: start-up or an import). The
+/// analysis of a body costs about its length (`import statistics`, whose
+/// imports compile regular expressions, spent 12 ms and a sixth of its
+/// instructions analyzing `re._parser._parse`'s 1,700 to reject them),
+/// and import-time code mostly runs once per process, so a large body
+/// needs work in proportion to its length; it can still compile at the
+/// ordinary threshold once the imports are done.
+fn import_size_budget(code: &CodeObject, admit_at: u32) -> u32 {
+    let len = code.instructions.len();
+    if compile_allowed(0, 1) || len <= IMPORT_LARGE_BODY {
+        return 0;
+    }
+    let scale = u32::try_from(len / IMPORT_LARGE_BODY + 1).unwrap_or(u32::MAX);
+    admit_at.saturating_mul(scale)
+}
+
 /// [`compile_allowed`] for the frame compiler (see `frame_jit`), which
 /// keeps tier 2's start-up and import budgets.
 pub(crate) fn frame_compile_allowed(counter: u32, threshold: u32) -> bool {
@@ -1311,11 +1334,19 @@ impl JitState {
                     if entry_pc == 0 {
                         entry.calls = entry.calls.wrapping_add(1);
                     }
-                    if entry.counter == self.threshold / 2 && self.engine.is_none() {
+                    // Halfway to the work this phase admits a compile at
+                    // (start-up and imports need far more first).
+                    let admit_at = if compile_allowed(0, self.threshold) {
+                        self.threshold
+                    } else {
+                        self.threshold.saturating_mul(IMPORT_THRESHOLD_FACTOR)
+                    };
+                    if entry.counter == admit_at / 2 && self.engine.is_none() {
                         spawn_codegen_prewarm();
                     }
                     if entry.counter < self.threshold
                         || !compile_allowed(entry.counter, self.threshold)
+                        || entry.counter < import_size_budget(code, admit_at)
                     {
                         return None;
                     }
@@ -1344,15 +1375,9 @@ impl JitState {
             long_calls = entry.backedges / calls >= LOOP_CALL_ITERATIONS;
         }
         // Threshold reached: compile (engine + cache borrowed disjointly).
-        if self.engine.is_none() {
-            self.engine = JitEngine::new();
-            if self.engine.is_none() {
-                // Host ISA unavailable — disable so we stop retrying.
-                self.enabled = false;
-                return None;
-            }
-        }
-        let engine = self.engine.as_mut()?;
+        // The engine is built once an analysis admits some code (see
+        // `JitEngine::compile_frame_lazily`).
+        let engine = &mut self.engine;
         let VmProbes {
             resolve_obj,
             ret_lane_of,
@@ -1675,7 +1700,8 @@ impl JitState {
             let mut slot_member =
                 |site: &AttrSiteMeta| attr_guard_of(site).is_some_and(|g| g.slot_layout != 0);
             ensure_obj_layout();
-            let r = engine.compile_frame_direct(
+            let r = JitEngine::compile_frame_lazily(
+                engine,
                 code,
                 &mut classify,
                 &mut jit_probes,
@@ -1701,6 +1727,14 @@ impl JitState {
                 first
             }
         };
+        if matches!(
+            res,
+            Err(weavepy_jit::JitVerdict::UnsupportedOpcode(why)) if why == weavepy_jit::NO_HOST_ENGINE
+        ) {
+            // Host ISA unavailable — disable so we stop retrying.
+            self.enabled = false;
+            return None;
+        }
         let (tier, out) = match res {
             Ok(cf) => {
                 self.stats.frames_compiled += 1;
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 3cadb8a9..e42278bc 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -597,8 +597,7 @@ impl LeafAttrCache {
         if self.0.get() == 0 {
             let t: Box<[LeafAttrEntry; LEAF_ATTRS_SMALL]> =
                 Box::new(std::array::from_fn(|_| LeafAttrEntry::EMPTY));
-            self.0
-                .set(Box::into_raw(t) as usize | LEAF_ATTRS_SMALL_TAG);
+            self.0.set(Box::into_raw(t) as usize | LEAF_ATTRS_SMALL_TAG);
         }
         let t = self.0.get();
         let len = if t & LEAF_ATTRS_SMALL_TAG != 0 {
@@ -768,7 +767,10 @@ mod leaf_attr_cache_tests {
             cache.get(first, 1),
             Some(LeafAttrKind::Value(Object::Str(value))) if SharedStr::ptr_eq(value, &text)
         ));
-        assert!(matches!(cache.get(rivals[1], 1), Some(LeafAttrKind::InstanceOnly)));
+        assert!(matches!(
+            cache.get(rivals[1], 1),
+            Some(LeafAttrKind::InstanceOnly)
+        ));
         assert_eq!(SharedStr::strong_count(&text), 2);
         drop(cache);
         assert_eq!(SharedStr::strong_count(&text), 1);

From 2734d9e3a7cc6de0d3afed9a239d8b13e5fa03ed Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 10:39:33 -0700
Subject: [PATCH 070/173] perf: fit a code object in mimalloc's 384-byte size
 class

A code object took 432 bytes (448 with its Arc's counts), and importing
asyncio decodes about 4,300 of them. Three changes bring it to 360:

- Each of the line, column and wire-mark tables was 32 bytes: a
  OnceLock'd box for the decoded vector and an optional range of the
  module's encoded buffer. A table is now 16 bytes: the range and one
  atomic word holding either the buffer's Arc (still encoded) or the
  decoded table, which takes over that count so a racing reader can
  still decode from the buffer.
- VmExt kept the payload's drop function beside its address; the
  payload now starts with it, so the slot is one word.
- co_name and co_qualname are pooled Names like the other name tables,
  16 bytes each instead of 24 plus a heap block. The code cache's native
  form (version 6) writes them through the module's identifier table.

With the earlier name pooling and leaf cache change, the live heap
after `import asyncio` is 15.00 MB against 15.68 MB.
---
 crates/weavepy-capi/src/abstract_.rs        |   2 +-
 crates/weavepy-compiler/src/lib.rs          | 375 +++++++++++++-------
 crates/weavepy-compiler/src/native_code.rs  |  99 ++++--
 crates/weavepy-vm/src/builtins.rs           |  12 +-
 crates/weavepy-vm/src/frame_jit.rs          |   2 +-
 crates/weavepy-vm/src/frozen_code_cache.rs  |   4 +-
 crates/weavepy-vm/src/lib.rs                |  12 +-
 crates/weavepy-vm/src/object.rs             |   6 +-
 crates/weavepy-vm/src/pycache.rs            |   2 +-
 crates/weavepy-vm/src/stdlib/marshal_mod.rs |   8 +-
 10 files changed, 332 insertions(+), 190 deletions(-)

diff --git a/crates/weavepy-capi/src/abstract_.rs b/crates/weavepy-capi/src/abstract_.rs
index 075d4c4a..5bf6031e 100644
--- a/crates/weavepy-capi/src/abstract_.rs
+++ b/crates/weavepy-capi/src/abstract_.rs
@@ -701,7 +701,7 @@ fn type_name(o: &Object) -> &'static str {
 fn callable_label(o: &Object) -> String {
     use Object as O;
     match o {
-        O::Function(f) => f.code().qualname.clone(),
+        O::Function(f) => f.code().qualname.to_string(),
         O::Builtin(b) => b.name.to_string(),
         O::Type(t) => format!("type:{}", t.name),
         O::BoundMethod(bm) => format!("bound:{}", callable_label(&bm.function)),
diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs
index a55917ba..aa557135 100644
--- a/crates/weavepy-compiler/src/lib.rs
+++ b/crates/weavepy-compiler/src/lib.rs
@@ -165,20 +165,27 @@ fn has_name(names: &[Name], name: &str) -> bool {
 /// cloned code object starts with an empty slot), never participates in
 /// equality, and is not serialized.
 ///
-/// The slot holds the payload's address (the VM's hot accessor reads one
-/// thin pointer) and the function that drops it, two words in all.
+/// The slot is one word: the address of the payload, which starts with
+/// the function that drops it (the VM's hot accessor reads one thin
+/// pointer).
 #[derive(Default)]
 pub struct VmExt {
-    /// The payload, a leaked `Box`, or null.
+    /// The payload, a leaked `Box<Payload<T>>`, or null.
     payload: std::sync::atomic::AtomicPtr<()>,
-    /// The payload's `drop_payload` instance, stored once `payload` is.
-    drop: std::sync::atomic::AtomicPtr<()>,
+}
+
+/// A [`VmExt`] payload: the value, after the function that drops the box.
+#[repr(C)]
+struct Payload<T> {
+    drop: fn(*mut ()),
+    value: T,
 }
 
 /// Drop a [`VmExt`] payload of type `T`.
 fn drop_payload<T>(payload: *mut ()) {
-    // SAFETY: `payload` is the `Box<T>` `VmExt::get_or_init` leaked.
-    drop(unsafe { Box::from_raw(payload.cast::<T>()) });
+    // SAFETY: `payload` is the `Box<Payload<T>>` `VmExt::get_or_init`
+    // leaked.
+    drop(unsafe { Box::from_raw(payload.cast::<Payload<T>>()) });
 }
 
 impl VmExt {
@@ -190,9 +197,9 @@ impl VmExt {
     #[inline]
     pub unsafe fn get<T>(&self) -> Option<&T> {
         let p = self.payload.load(std::sync::atomic::Ordering::Acquire);
-        // SAFETY: a non-null payload is a live `T` (the caller's contract)
-        // that lives as long as the slot.
-        (!p.is_null()).then(|| unsafe { &*p.cast::<T>() })
+        // SAFETY: a non-null payload is a live `Payload<T>` (the caller's
+        // contract) that lives as long as the slot.
+        (!p.is_null()).then(|| unsafe { &(*p.cast::<Payload<T>>()).value })
     }
 
     /// The payload, set from `init` if there is none yet. Racing
@@ -207,26 +214,23 @@ impl VmExt {
         if let Some(payload) = unsafe { self.get::<T>() } {
             return payload;
         }
-        let fresh = Box::into_raw(Box::new(init())).cast::<()>();
+        let fresh = Box::into_raw(Box::new(Payload {
+            drop: drop_payload::<T>,
+            value: init(),
+        }))
+        .cast::<()>();
         match self.payload.compare_exchange(
             std::ptr::null_mut(),
             fresh,
             Ordering::AcqRel,
             Ordering::Acquire,
         ) {
-            Ok(_) => {
-                // Read only by `drop`, which has the slot to itself.
-                self.drop.store(
-                    drop_payload::<T> as fn(*mut ()) as *mut (),
-                    Ordering::Release,
-                );
-                // SAFETY: just published; it lives as long as the slot.
-                unsafe { &*fresh.cast::<T>() }
-            }
+            // SAFETY: just published; it lives as long as the slot.
+            Ok(_) => unsafe { &(*fresh.cast::<Payload<T>>()).value },
             Err(won) => {
                 drop_payload::<T>(fresh);
-                // SAFETY: the winner's `T` (the caller's contract).
-                unsafe { &*won.cast::<T>() }
+                // SAFETY: the winner's `Payload<T>` (the caller's contract).
+                unsafe { &(*won.cast::<Payload<T>>()).value }
             }
         }
     }
@@ -242,11 +246,11 @@ impl VmExt {
 
 impl Drop for VmExt {
     fn drop(&mut self) {
-        let (payload, drop_fn) = (*self.payload.get_mut(), *self.drop.get_mut());
-        if !payload.is_null() && !drop_fn.is_null() {
-            // SAFETY: `drop_fn` was stored from a `fn(*mut ())` for this
-            // payload's type.
-            let drop_fn = unsafe { std::mem::transmute::<*mut (), fn(*mut ())>(drop_fn) };
+        let payload = *self.payload.get_mut();
+        if !payload.is_null() {
+            // SAFETY: every payload starts with its own drop function
+            // (`Payload` is `repr(C)`).
+            let drop_fn = unsafe { *payload.cast::<fn(*mut ())>() };
             drop_fn(payload);
         }
     }
@@ -581,13 +585,13 @@ impl std::fmt::Debug for JitHint {
 /// `PyCodeObject` we need to emulate.
 #[derive(Debug, Clone, Default, PartialEq)]
 pub struct CodeObject {
-    pub name: String,
+    pub name: Name,
     /// Dotted qualified name (PEP 3155), computed at compile time from the
     /// lexical scope nesting: `outer.<locals>.inner` for a function nested
     /// in `outer`, `C.method` for a method of class `C`. Equals `name` for
     /// module-level definitions. Drives `function.__qualname__` /
     /// `type.__qualname__` (and thus reprs, error messages, and pickling).
-    pub qualname: String,
+    pub qualname: Name,
     /// Source filename or `<string>`. Used for diagnostics only. Shared:
     /// every code object compiled or decoded from one module holds the
     /// same allocation.
@@ -784,10 +788,10 @@ impl PartialEq for CodeRare {
 }
 
 // A decoded module holds a code object per function, class body and
-// comprehension, each behind an `Arc`'s two counts: at 432 bytes or
-// fewer, the allocation fits mimalloc's 448-byte size class.
+// comprehension, each behind an `Arc`'s two counts: at 368 bytes or
+// fewer, the allocation fits mimalloc's 384-byte size class.
 #[cfg(target_pointer_width = "64")]
-const _: () = assert!(std::mem::size_of::<CodeObject>() <= 432);
+const _: () = assert!(std::mem::size_of::<CodeObject>() <= 368);
 
 impl CodeObject {
     /// See [`RareFields::no_interrupt_jumps`].
@@ -822,49 +826,201 @@ pub struct WireOverrides {
     pub exec_error: Option<String>,
 }
 
-/// A per-instruction source-column span (PEP-657). `col`/`end_col` are
-/// 0-based UTF-8 byte offsets within their respective source lines, and
+/// A table decoded on first read from the code cache's encoding of it
+/// (see `native_code`), or a vector: the state behind [`LineTable`],
+/// [`ColTable`] and [`WireMarks`].
+///
+/// Two words: the encoding's range of the buffer every code object of a
+/// module shares, and one word of state. State `0` is an empty table not
+/// yet materialized; an address with [`LAZY_ENCODED`] set is the shared
+/// buffer's `Arc` (one count), still encoded; any other address is the
+/// decoded table, which takes over that count. (It keeps the buffer
+/// alive while a racing reader may still decode from it, until the table
+/// is dropped or borrowed mutably.)
+struct LazyTable<T> {
+    state: std::sync::atomic::AtomicUsize,
+    start: u32,
+    end: u32,
+    _items: std::marker::PhantomData<Vec<T>>,
+}
+
+/// A module's encoded position tables, filled once the decoding is done
+/// (see `native_code::decode`).
+pub(crate) type TableBuffer = std::sync::OnceLock<Box<[u8]>>;
+
+/// The tag of an encoded [`LazyTable`]'s state.
+const LAZY_ENCODED: usize = 1;
+
+/// A decoded [`LazyTable`].
+struct Decoded<T> {
+    items: Vec<T>,
+    /// The buffer the table was decoded from, if any.
+    source: Option<std::sync::Arc<TableBuffer>>,
+}
+
+// SAFETY: the state is either nothing, an `Arc` of a shareable buffer, or
+// an owned `Box<Decoded<T>>` published through the atomic.
+unsafe impl<T: Send> Send for LazyTable<T> {}
+unsafe impl<T: Send + Sync> Sync for LazyTable<T> {}
+
+impl<T> LazyTable<T> {
+    fn new(state: usize, start: u32, end: u32) -> Self {
+        LazyTable {
+            state: std::sync::atomic::AtomicUsize::new(state),
+            start,
+            end,
+            _items: std::marker::PhantomData,
+        }
+    }
+
+    fn from_vec(items: Vec<T>) -> Self {
+        let decoded = Box::new(Decoded {
+            items,
+            source: None,
+        });
+        Self::new(Box::into_raw(decoded) as usize, 0, 0)
+    }
+
+    fn encoded(encoded: Encoded) -> Self {
+        let Encoded { buffer, start, end } = encoded;
+        Self::new(
+            std::sync::Arc::into_raw(buffer) as usize | LAZY_ENCODED,
+            start,
+            end,
+        )
+    }
+
+    /// The table, decoded with `decode` (or empty when that fails) if
+    /// this is its first read.
+    #[inline]
+    fn get(&self, decode: fn(&[u8]) -> Option<Vec<T>>) -> &Vec<T> {
+        use std::sync::atomic::Ordering;
+        let state = self.state.load(Ordering::Acquire);
+        if state != 0 && state & LAZY_ENCODED == 0 {
+            // SAFETY: a decoded table lives as long as `self`.
+            return unsafe { &(*(state as *const Decoded<T>)).items };
+        }
+        self.decode(state, decode)
+    }
+
+    #[cold]
+    #[inline(never)]
+    fn decode(&self, state: usize, decode: fn(&[u8]) -> Option<Vec<T>>) -> &Vec<T> {
+        use std::sync::atomic::Ordering;
+        let buffer = (state & !LAZY_ENCODED) as *const TableBuffer;
+        let (items, source) = if buffer.is_null() {
+            (Vec::new(), None)
+        } else {
+            // SAFETY: the state holds a count of this `Arc`, which the
+            // decoded table takes over if it is published (and which a
+            // published table keeps alive for any racing reader).
+            let source = unsafe { std::sync::Arc::from_raw(buffer) };
+            let items = source
+                .get()
+                .and_then(|b| b.get(self.start as usize..self.end as usize))
+                .and_then(decode)
+                .unwrap_or_default();
+            (items, Some(source))
+        };
+        let fresh = Box::into_raw(Box::new(Decoded { items, source }));
+        match self.state.compare_exchange(
+            state,
+            fresh as usize,
+            Ordering::AcqRel,
+            Ordering::Acquire,
+        ) {
+            // SAFETY: just published; it lives as long as `self`.
+            Ok(_) => unsafe { &(*fresh).items },
+            Err(won) => {
+                // SAFETY: never published; the count it holds is the
+                // state's, which the winner took over.
+                let lost = unsafe { Box::from_raw(fresh) };
+                if let Some(source) = lost.source {
+                    let _ = std::sync::Arc::into_raw(source);
+                }
+                // SAFETY: the winner's decoded table lives as long as
+                // `self`.
+                unsafe { &(*(won as *const Decoded<T>)).items }
+            }
+        }
+    }
+
+    fn get_mut(&mut self, decode: fn(&[u8]) -> Option<Vec<T>>) -> &mut Vec<T> {
+        self.get(decode);
+        let state = *self.state.get_mut();
+        // SAFETY: `get` left a decoded table, and the exclusive borrow
+        // means no reader can still be decoding from its source.
+        let decoded = unsafe { &mut *(state as *mut Decoded<T>) };
+        decoded.source = None;
+        &mut decoded.items
+    }
+}
+
+impl<T> Drop for LazyTable<T> {
+    fn drop(&mut self) {
+        let state = *self.state.get_mut();
+        if state == 0 {
+            return;
+        }
+        // SAFETY: the state owns what it points at (see the type docs).
+        unsafe {
+            if state & LAZY_ENCODED != 0 {
+                drop(std::sync::Arc::from_raw(
+                    (state & !LAZY_ENCODED) as *const TableBuffer,
+                ));
+            } else {
+                drop(Box::from_raw(state as *mut Decoded<T>));
+            }
+        }
+    }
+}
+
+impl<T> Default for LazyTable<T> {
+    fn default() -> Self {
+        Self::new(0, 0, 0)
+    }
+}
+
+impl<T: Clone> Clone for LazyTable<T> {
+    fn clone(&self) -> Self {
+        let state = self.state.load(std::sync::atomic::Ordering::Acquire);
+        if state == 0 {
+            Self::default()
+        } else if state & LAZY_ENCODED != 0 {
+            // SAFETY: the state holds a count of this `Arc`; the clone
+            // takes another.
+            unsafe {
+                std::sync::Arc::increment_strong_count(
+                    (state & !LAZY_ENCODED) as *const TableBuffer,
+                );
+            }
+            Self::new(state, self.start, self.end)
+        } else {
+            // SAFETY: a decoded table lives as long as `self`.
+            Self::from_vec(unsafe { &(*(state as *const Decoded<T>)).items }.clone())
+        }
+    }
+}
+
 /// A code object's column spans (see [`CodeObject::coltable`]): a
 /// vector, or the code cache's encoding of one, decoded on first use.
 /// Most code never reports a column (only tracebacks and
 /// `co_positions()` read them), so a module loaded from the cache skips
 /// the work until something does.
 #[derive(Clone, Default)]
-pub struct ColTable {
-    // Boxed so that the (usual) undecoded table costs one word here.
-    #[allow(clippy::box_collection)]
-    spans: std::sync::OnceLock<Box<Vec<ColSpan>>>,
-    encoded: Option<Encoded>,
-}
+pub struct ColTable(LazyTable<ColSpan>);
 
 impl ColTable {
     /// The spans `encoded` holds in the code cache's form (see
     /// `native_code`), decoded when first read.
     pub(crate) fn encoded(encoded: Encoded) -> Self {
-        ColTable {
-            spans: std::sync::OnceLock::new(),
-            encoded: Some(encoded),
-        }
-    }
-
-    fn spans(&self) -> &Vec<ColSpan> {
-        self.spans.get_or_init(|| {
-            Box::new(
-                self.encoded
-                    .as_ref()
-                    .and_then(|e| native_code::decode_coltable(e.bytes()))
-                    .unwrap_or_default(),
-            )
-        })
+        ColTable(LazyTable::encoded(encoded))
     }
 }
 
 impl From<Vec<ColSpan>> for ColTable {
     fn from(spans: Vec<ColSpan>) -> Self {
-        ColTable {
-            spans: std::sync::OnceLock::from(Box::new(spans)),
-            encoded: None,
-        }
+        ColTable(LazyTable::from_vec(spans))
     }
 }
 
@@ -872,15 +1028,13 @@ impl std::ops::Deref for ColTable {
     type Target = Vec<ColSpan>;
 
     fn deref(&self) -> &Vec<ColSpan> {
-        self.spans()
+        self.0.get(native_code::decode_coltable)
     }
 }
 
 impl std::ops::DerefMut for ColTable {
     fn deref_mut(&mut self) -> &mut Vec<ColSpan> {
-        self.spans();
-        self.encoded = None;
-        self.spans.get_mut().expect("decoded above")
+        self.0.get_mut(native_code::decode_coltable)
     }
 }
 
@@ -903,32 +1057,13 @@ impl std::fmt::Debug for ColTable {
 /// a module loaded from the cache keeps the compact form until something
 /// does, instead of four bytes per instruction.
 #[derive(Clone, Default)]
-pub struct LineTable {
-    // Boxed so that the (usual) undecoded table costs one word here.
-    #[allow(clippy::box_collection)]
-    lines: std::sync::OnceLock<Box<Vec<u32>>>,
-    encoded: Option<Encoded>,
-}
+pub struct LineTable(LazyTable<u32>);
 
 impl LineTable {
     /// The lines `encoded` holds in the code cache's form (see
     /// `native_code`), decoded when first read.
     pub(crate) fn encoded(encoded: Encoded) -> Self {
-        LineTable {
-            lines: std::sync::OnceLock::new(),
-            encoded: Some(encoded),
-        }
-    }
-
-    fn lines(&self) -> &Vec<u32> {
-        self.lines.get_or_init(|| {
-            Box::new(
-                self.encoded
-                    .as_ref()
-                    .and_then(|e| native_code::decode_linetable(e.bytes()))
-                    .unwrap_or_default(),
-            )
-        })
+        LineTable(LazyTable::encoded(encoded))
     }
 }
 
@@ -936,59 +1071,36 @@ impl LineTable {
 /// encoding: a range of a buffer that every code object decoded from one
 /// module shares (see `native_code::decode`), so a module's tables take
 /// one allocation rather than three per code object.
-#[derive(Clone)]
 pub(crate) struct Encoded {
     /// Set once the module's decoding finishes, before any table is read.
-    buffer: std::sync::Arc<std::sync::OnceLock<Box<[u8]>>>,
+    buffer: std::sync::Arc<TableBuffer>,
     start: u32,
     end: u32,
 }
 
 impl Encoded {
-    pub(crate) fn new(
-        buffer: std::sync::Arc<std::sync::OnceLock<Box<[u8]>>>,
-        start: u32,
-        end: u32,
-    ) -> Self {
+    pub(crate) fn new(buffer: std::sync::Arc<TableBuffer>, start: u32, end: u32) -> Self {
         Encoded { buffer, start, end }
     }
-
-    fn bytes(&self) -> &[u8] {
-        self.buffer
-            .get()
-            .and_then(|b| b.get(self.start as usize..self.end as usize))
-            .unwrap_or_default()
-    }
 }
 
 /// A code object's wire marks (see [`CodeObject::wire_marks`]): a vector,
 /// or the code cache's encoding of one (two marks a byte), decoded on
 /// first use. Only the CPython wire form of the code reads them.
 #[derive(Clone, Default)]
-pub struct WireMarks {
-    // Boxed so that the (usual) undecoded marks cost one word here.
-    #[allow(clippy::box_collection)]
-    marks: std::sync::OnceLock<Box<Vec<u8>>>,
-    encoded: Option<Encoded>,
-}
+pub struct WireMarks(LazyTable<u8>);
 
 impl WireMarks {
     /// The marks `encoded` holds in the code cache's form (see
     /// `native_code`), decoded when first read.
     pub(crate) fn encoded(encoded: Encoded) -> Self {
-        WireMarks {
-            marks: std::sync::OnceLock::new(),
-            encoded: Some(encoded),
-        }
+        WireMarks(LazyTable::encoded(encoded))
     }
 }
 
 impl From<Vec<u8>> for WireMarks {
     fn from(marks: Vec<u8>) -> Self {
-        WireMarks {
-            marks: std::sync::OnceLock::from(Box::new(marks)),
-            encoded: None,
-        }
+        WireMarks(LazyTable::from_vec(marks))
     }
 }
 
@@ -996,14 +1108,7 @@ impl std::ops::Deref for WireMarks {
     type Target = [u8];
 
     fn deref(&self) -> &[u8] {
-        self.marks.get_or_init(|| {
-            Box::new(
-                self.encoded
-                    .as_ref()
-                    .and_then(|e| native_code::decode_wire_marks(e.bytes()))
-                    .unwrap_or_default(),
-            )
-        })
+        self.0.get(native_code::decode_wire_marks)
     }
 }
 
@@ -1021,10 +1126,7 @@ impl std::fmt::Debug for WireMarks {
 
 impl From<Vec<u32>> for LineTable {
     fn from(lines: Vec<u32>) -> Self {
-        LineTable {
-            lines: std::sync::OnceLock::from(Box::new(lines)),
-            encoded: None,
-        }
+        LineTable(LazyTable::from_vec(lines))
     }
 }
 
@@ -1032,15 +1134,13 @@ impl std::ops::Deref for LineTable {
     type Target = Vec<u32>;
 
     fn deref(&self) -> &Vec<u32> {
-        self.lines()
+        self.0.get(native_code::decode_linetable)
     }
 }
 
 impl std::ops::DerefMut for LineTable {
     fn deref_mut(&mut self) -> &mut Vec<u32> {
-        self.lines();
-        self.encoded = None;
-        self.lines.get_mut().expect("decoded above")
+        self.0.get_mut(native_code::decode_linetable)
     }
 }
 
@@ -1056,6 +1156,8 @@ impl std::fmt::Debug for LineTable {
     }
 }
 
+/// A per-instruction source-column span (PEP-657). `col`/`end_col` are
+/// 0-based UTF-8 byte offsets within their respective source lines, and
 /// are `-1` when the column was not tracked. `end_lineno` is `0` when
 /// unknown (callers fall back to the instruction's start line).
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -2686,6 +2788,7 @@ impl Compiler {
         let mut co = CodeObject::default();
         // Default qualname == name; nested scopes overwrite this via
         // `compute_child_qualname` once the parent context is known.
+        let name = Name::from(name);
         co.qualname = name.clone();
         co.name = name;
         co.filename = filename;
@@ -2836,7 +2939,7 @@ impl Compiler {
         if force_global && self.explicit_globals.contains(name) {
             return name.to_owned();
         }
-        let mut base = self.co.qualname.clone();
+        let mut base = self.co.qualname.to_string();
         if matches!(self.kind, CodeKind::Function) {
             base.push_str(".<locals>");
         }
@@ -2855,7 +2958,7 @@ impl Compiler {
         if matches!(self.kind, CodeKind::Module) {
             return String::new();
         }
-        let mut base = self.co.qualname.clone();
+        let mut base = self.co.qualname.to_string();
         if matches!(self.kind, CodeKind::Function) && self.annotation_qualname.is_none() {
             base.push_str(".<locals>");
         }
@@ -5775,7 +5878,7 @@ impl Compiler {
                 .insert("__classdict__".to_owned(), Binding::Free);
             inner.free_order.push("__classdict__".to_owned());
         }
-        inner.co.qualname = self.compute_child_qualname(hidden_name);
+        inner.co.qualname = self.compute_child_qualname(hidden_name).into();
         inner.co.arg_count = arg_count;
         inner.co.varnames = hidden_params.iter().map(Name::from).collect();
         inner.current_line = entry_line.unwrap_or(self.current_line);
@@ -6259,7 +6362,7 @@ impl Compiler {
                 .insert("__classdict__".to_owned(), Binding::Free);
             inner.free_order.push("__classdict__".to_owned());
         }
-        inner.co.qualname = self.compute_child_qualname(display);
+        inner.co.qualname = self.compute_child_qualname(display).into();
         inner.co.arg_count = arg_count;
         inner.co.posonly_count = posonly_count;
         inner.co.kwonly_count = kwonly_count;
@@ -6515,7 +6618,7 @@ impl Compiler {
                 .insert("__classdict__".to_owned(), Binding::Free);
             inner.free_order.push("__classdict__".to_owned());
         }
-        inner.co.qualname = self.compute_annotation_child_qualname(name);
+        inner.co.qualname = self.compute_annotation_child_qualname(name).into();
         // A lambda or comprehension inside the annotation scope is
         // named from *this* scope (`compiler_set_qualname` looks
         // through the annotation parent).
@@ -6999,7 +7102,7 @@ impl Compiler {
         // A class body never reports `CO_NESTED` itself (no
         // `CO_OPTIMIZED`), but its methods inherit the nesting.
         inner.co.is_nested = self.child_is_nested();
-        inner.co.qualname = self.compute_child_qualname(name);
+        inner.co.qualname = self.compute_child_qualname(name).into();
         inner.current_line = entry_line.unwrap_or(self.current_line);
         // CPython only gives a class body the `__class__` closure cell —
         // and the trailing `__classcell__` store — when a method actually
@@ -7271,7 +7374,7 @@ impl Compiler {
             inner.emit_load_name("__name__");
             inner.emit_store_name("__module__");
             let qualname_str = inner.co.qualname.clone();
-            let qualname_const = inner.intern_constant(Constant::Str(qualname_str));
+            let qualname_const = inner.intern_constant(Constant::Str(qualname_str.to_string()));
             inner.emit(OpCode::LoadConst, qualname_const);
             inner.emit_store_name("__qualname__");
             // A `nonlocal __firstlineno__` declaration in the class body
@@ -11698,7 +11801,7 @@ impl Compiler {
         // PEP 3155: a comprehension scope gets a dotted qualname like any
         // other nested scope (`C.m.<locals>.<genexpr>`); CPython's
         // `compiler_set_qualname` doesn't special-case comprehensions.
-        inner.co.qualname = self.compute_child_qualname(name);
+        inner.co.qualname = self.compute_child_qualname(name).into();
         inner.co.arg_count = 1;
         inner.co.varnames.push(Name::from(".0".to_owned()));
         inner.bindings.insert(".0".to_owned(), Binding::Local);
diff --git a/crates/weavepy-compiler/src/native_code.rs b/crates/weavepy-compiler/src/native_code.rs
index 5b839986..f2748cee 100644
--- a/crates/weavepy-compiler/src/native_code.rs
+++ b/crates/weavepy-compiler/src/native_code.rs
@@ -18,7 +18,7 @@ use crate::bytecode::{CacheTable, Instruction, OpCode};
 use crate::{CodeObject, ColSpan, Constant, ExcHandler, Name};
 
 /// The layout revision; bump it whenever the encoding changes.
-pub const VERSION: u8 = 5;
+pub const VERSION: u8 = 6;
 
 /// Encode `code` (and its nested code objects). `None` for a code object
 /// the format doesn't carry: one with raw CPython wire overrides, or with
@@ -173,20 +173,24 @@ impl Writer {
         }
     }
 
-    /// A name table: its length, then per name its number plus one when
-    /// the module wrote it before, else a zero and its text (which gives
-    /// it the next number).
+    /// An identifier: its number plus one when the module wrote it
+    /// before, else a zero and its text (which gives it the next number).
+    fn name(&mut self, s: &Name) {
+        if let Some(&i) = self.1.get(s.as_str()) {
+            self.uint(u64::from(i) + 1);
+        } else {
+            let i = self.1.len() as u32;
+            self.1.insert(s.as_str().to_owned(), i);
+            self.uint(0);
+            self.bytes(s.as_bytes());
+        }
+    }
+
+    /// A name table: its length, then its identifiers (see [`Self::name`]).
     fn names(&mut self, v: &[Name]) {
         self.uint(v.len() as u64);
         for s in v {
-            if let Some(&i) = self.1.get(s.as_str()) {
-                self.uint(u64::from(i) + 1);
-            } else {
-                let i = self.1.len() as u32;
-                self.1.insert(s.as_str().to_owned(), i);
-                self.uint(0);
-                self.bytes(s.as_bytes());
-            }
+            self.name(s);
         }
     }
 
@@ -248,8 +252,8 @@ impl Writer {
         if c.wire.is_some() {
             return None;
         }
-        self.bytes(c.name.as_bytes());
-        self.bytes(c.qualname.as_bytes());
+        self.name(&c.name);
+        self.name(&c.qualname);
         self.uint(c.instructions.len() as u64);
         for ins in &c.instructions {
             self.byte(ins.op as u8);
@@ -453,20 +457,24 @@ impl Reader<'_> {
         Some(v)
     }
 
+    /// An identifier [`Writer::name`] wrote.
+    fn name(&mut self, module: &mut Module) -> Option<Name> {
+        Some(match self.uint()? {
+            0 => {
+                let name = Name::new(std::str::from_utf8(self.raw()?).ok()?);
+                module.names.push(name.clone());
+                name
+            }
+            i => module.names.get(usize::try_from(i - 1).ok()?)?.clone(),
+        })
+    }
+
     /// A name table [`Writer::names`] wrote.
     fn names(&mut self, module: &mut Module) -> Option<Vec<Name>> {
         let n = self.len()?;
         let mut v = Vec::with_capacity(n);
         for _ in 0..n {
-            let name = match self.uint()? {
-                0 => {
-                    let name = Name::new(std::str::from_utf8(self.raw()?).ok()?);
-                    module.names.push(name.clone());
-                    name
-                }
-                i => module.names.get(usize::try_from(i - 1).ok()?)?.clone(),
-            };
-            v.push(name);
+            v.push(self.name(module)?);
         }
         Some(v)
     }
@@ -534,8 +542,8 @@ impl Reader<'_> {
         if depth > MAX_NESTING {
             return None;
         }
-        let name = self.string()?;
-        let qualname = self.string()?;
+        let name = self.name(module)?;
+        let qualname = self.name(module)?;
         let n = self.len()?;
         let mut instructions = Vec::with_capacity(n);
         for _ in 0..n {
@@ -661,8 +669,8 @@ mod tests {
     #[test]
     fn round_trips_constants_and_metadata() {
         let inner = CodeObject {
-            name: "f".to_owned(),
-            qualname: "C.f".to_owned(),
+            name: "f".into(),
+            qualname: "C.f".into(),
             filename: "m.py".into(),
             instructions: vec![
                 Instruction::new(OpCode::Resume, 0),
@@ -680,8 +688,8 @@ mod tests {
             ..CodeObject::default()
         };
         let mut code = CodeObject {
-            name: "<module>".to_owned(),
-            qualname: "<module>".to_owned(),
+            name: "<module>".into(),
+            qualname: "<module>".into(),
             filename: "m.py".into(),
             instructions: vec![Instruction::new(OpCode::Nop, 0)],
             constants: vec![
@@ -731,6 +739,37 @@ mod tests {
         assert_eq!(*inner.linetable, [1, 2, 2]);
     }
 
+    #[test]
+    fn encoded_tables_clone_and_mutate_independently() {
+        let code = CodeObject {
+            instructions: vec![Instruction::new(OpCode::Nop, 0); 3],
+            linetable: vec![1, 2, 2].into(),
+            coltable: vec![ColSpan::default(); 3].into(),
+            wire_marks: vec![0, 3, 0].into(),
+            ..CodeObject::default()
+        };
+        let back = decode(&encode(&code).expect("encodable"), "m.py").expect("decodable");
+        // A clone of a table still encoded shares the encoding.
+        let copy = back.clone();
+        assert_eq!(*copy.linetable, [1, 2, 2]);
+        assert_eq!(*back.linetable, [1, 2, 2]);
+        // Mutating a table decodes it first, and leaves the others alone.
+        let mut owned = back.clone();
+        owned.linetable.push(3);
+        owned.coltable.pop();
+        assert_eq!(*owned.linetable, [1, 2, 2, 3]);
+        assert_eq!(owned.coltable.len(), 2);
+        assert_eq!(*back.linetable, [1, 2, 2]);
+        assert_eq!(back.coltable.len(), 3);
+        assert_eq!(&*copy.wire_marks, &[0, 3, 0]);
+        // A clone of a decoded table copies it.
+        let again = owned.clone();
+        drop(owned);
+        assert_eq!(*again.linetable, [1, 2, 2, 3]);
+        // A default table reads as empty.
+        assert!(CodeObject::default().linetable.is_empty());
+    }
+
     #[test]
     fn round_trips_position_runs() {
         let span = |end_lineno, col, end_col| ColSpan {
@@ -773,7 +812,7 @@ mod tests {
     #[test]
     fn rejects_truncated_or_foreign_input() {
         let code = CodeObject {
-            name: "<module>".to_owned(),
+            name: "<module>".into(),
             instructions: vec![Instruction::new(OpCode::Nop, 0)],
             constants: vec![Constant::Str("x".repeat(40))],
             ..CodeObject::default()
diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 574692a1..ee88aead 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -4566,8 +4566,8 @@ fn code_replace(args: &[Object], kwargs: &[(String, Object)]) -> Result<Object,
     let mut requested_nlocals: Option<u32> = None;
     for (k, v) in kwargs {
         match k.as_str() {
-            "co_name" => nc.name = want_str(v, "co_name")?,
-            "co_qualname" => nc.qualname = want_str(v, "co_qualname")?,
+            "co_name" => nc.name = want_str(v, "co_name")?.into(),
+            "co_qualname" => nc.qualname = want_str(v, "co_qualname")?.into(),
             "co_filename" => nc.filename = want_str(v, "co_filename")?.into(),
             "co_argcount" => nc.arg_count = want_u32(v, "co_argcount")?,
             "co_posonlyargcount" => nc.posonly_count = want_u32(v, "co_posonlyargcount")?,
@@ -4724,8 +4724,8 @@ pub fn foreign_code_object(
     const CO_ITERABLE_COROUTINE: u32 = 0x0100;
     const CO_ASYNC_GENERATOR: u32 = 0x0200;
     let mut nc = weavepy_compiler::CodeObject {
-        name,
-        qualname,
+        name: name.into(),
+        qualname: qualname.into(),
         filename: filename.into(),
         varnames: names_of(varnames),
         arg_count,
@@ -4923,8 +4923,8 @@ pub(crate) fn code_type_call(
     const CO_ITERABLE_COROUTINE: u32 = 0x0100;
     const CO_ASYNC_GENERATOR: u32 = 0x0200;
     let mut nc = weavepy_compiler::CodeObject {
-        name,
-        qualname,
+        name: name.into(),
+        qualname: qualname.into(),
         filename: filename.into(),
         constants,
         names: names_of(names),
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 834f0bcb..2e27f1e6 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -5942,7 +5942,7 @@ fn compile_with(
         nlocals,
         fails: entries.iter().map(|_| AtomicU8::new(0)).collect(),
         entries: entries.into_iter().map(AtomicBool::new).collect(),
-        name: code.qualname.clone(),
+        name: code.qualname.to_string(),
         ops: code.instructions.iter().map(|i| i.op).collect(),
         depths: depths.into(),
         need: stack_need(depths),
diff --git a/crates/weavepy-vm/src/frozen_code_cache.rs b/crates/weavepy-vm/src/frozen_code_cache.rs
index 62d3af8a..9a0a6ebc 100644
--- a/crates/weavepy-vm/src/frozen_code_cache.rs
+++ b/crates/weavepy-vm/src/frozen_code_cache.rs
@@ -231,7 +231,7 @@ mod tests {
     fn cache_returns_fresh_copies() {
         clear();
         let mut code = CodeObject::default();
-        code.name = "foo".to_owned();
+        code.name = "foo".into();
         insert("foo", &code);
         let got = get("foo").expect("hit");
         assert_eq!(got.name, "foo");
@@ -243,7 +243,7 @@ mod tests {
         use weavepy_compiler::{CacheTable, InlineCache};
         clear();
         let mut code = CodeObject::default();
-        code.name = "warmed".to_owned();
+        code.name = "warmed".into();
         code.caches = CacheTable::with_len(2);
         code.caches.set(0, InlineCache::BinOpAddInt);
         insert("warmed", &code);
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 60c04817..de2f2e0f 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -52410,7 +52410,7 @@ impl Interpreter {
         };
         let name = match args.get(2) {
             Some(Object::Str(s)) => s.to_string(),
-            Some(Object::None) | None => code.name.clone(),
+            Some(Object::None) | None => code.name.to_string(),
             Some(other) => {
                 return Err(type_error(format!(
                     "function() argument 'name' must be str or None, not {}",
@@ -56423,7 +56423,7 @@ impl Interpreter {
             f.slot("__qualname__")
                 .as_ref()
                 .map(Object::to_str)
-                .unwrap_or_else(|| code.qualname.clone())
+                .unwrap_or_else(|| code.qualname.to_string())
         };
         if !missing_positional.is_empty() {
             return Err(type_error(format_missing_arguments(
@@ -56445,7 +56445,7 @@ impl Interpreter {
                 .slot("__qualname__")
                 .as_ref()
                 .map(Object::to_str)
-                .unwrap_or_else(|| code.qualname.clone());
+                .unwrap_or_else(|| code.qualname.to_string());
             return Err(type_error(format_missing_arguments(
                 &error_name,
                 "keyword-only",
@@ -67020,7 +67020,7 @@ pub(crate) fn callable_function_str(callable: &Object) -> Option<String> {
                 .slot("__qualname__")
                 .as_ref()
                 .map(Object::to_str)
-                .unwrap_or_else(|| f.code().qualname.clone());
+                .unwrap_or_else(|| f.code().qualname.to_string());
             let module = f.slot("__module__").as_ref().map(Object::to_str);
             match module {
                 Some(m) if !m.is_empty() && m != "builtins" => Some(format!("{m}.{qual}()")),
@@ -74350,7 +74350,7 @@ fn set_function_attribute(func: Object, value: Object, flag: u32) -> Result<Obje
                     let owner_qualname = f
                         .slot("__qualname__")
                         .map(|q| q.to_str())
-                        .unwrap_or_else(|| f.code.borrow().qualname.clone());
+                        .unwrap_or_else(|| f.code.borrow().qualname.to_string());
                     annotate.set_slot(
                         "__qualname__",
                         Object::from_str(format!("{owner_qualname}.__annotate__")),
@@ -74454,7 +74454,7 @@ fn new_function(
         .get(&DictKey(fn_slot_keys()[1].clone()))
         .cloned();
     let f = PyFunction {
-        name: code.name.clone(),
+        name: code.name.to_string(),
         code: RefCell::new(code),
         globals: globals.clone(),
         // The defining frame's builtins, not a fresh resolution: the same
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 0901b6c0..5649d9bb 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -11671,7 +11671,7 @@ impl Object {
                     .slot("__qualname__")
                     .as_ref()
                     .map(Object::to_str)
-                    .unwrap_or_else(|| f.code().qualname.clone());
+                    .unwrap_or_else(|| f.code().qualname.to_string());
                 format!("<function {} at 0x{:x}>", qual, Rc::as_ptr(f) as usize)
             }
             Object::Builtin(b) => {
@@ -11717,7 +11717,7 @@ impl Object {
                         .slot("__qualname__")
                         .as_ref()
                         .map(Object::to_str)
-                        .unwrap_or_else(|| f.code().qualname.clone()),
+                        .unwrap_or_else(|| f.code().qualname.to_string()),
                     // A C function bound to its receiver is CPython's
                     // `builtin_function_or_method`, whose repr is
                     // `<built-in method split of str object at 0x…>`
@@ -12084,7 +12084,7 @@ impl Object {
                                 .slot("__qualname__")
                                 .as_ref()
                                 .map(Object::to_str)
-                                .unwrap_or_else(|| f.code().qualname.clone());
+                                .unwrap_or_else(|| f.code().qualname.to_string());
                             let module = f
                                 .slot("__module__")
                                 .or_else(|| {
diff --git a/crates/weavepy-vm/src/pycache.rs b/crates/weavepy-vm/src/pycache.rs
index 25743b2b..98c0de93 100644
--- a/crates/weavepy-vm/src/pycache.rs
+++ b/crates/weavepy-vm/src/pycache.rs
@@ -469,7 +469,7 @@ mod ownership_tests {
     #[test]
     fn relocation_preserves_shared_code_and_nested_tuple_constants() {
         let child = Arc::new(CodeObject {
-            name: "child".to_owned(),
+            name: "child".into(),
             filename: "original.py".into(),
             ..CodeObject::default()
         });
diff --git a/crates/weavepy-vm/src/stdlib/marshal_mod.rs b/crates/weavepy-vm/src/stdlib/marshal_mod.rs
index 1f740652..b9018d4c 100644
--- a/crates/weavepy-vm/src/stdlib/marshal_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/marshal_mod.rs
@@ -1192,8 +1192,8 @@ impl Scan {
         {
             self.identifiers.insert(name.to_owned());
         }
-        self.identifiers.insert(co.name.clone());
-        self.qualnames_done.insert(co.qualname.clone());
+        self.identifiers.insert(co.name.to_string());
+        self.qualnames_done.insert(co.qualname.to_string());
     }
 
     /// Strings nested in tuple/frozenset constants: plain literals,
@@ -1738,8 +1738,8 @@ impl<'a> MarshalReader<'a> {
         // (test_capi.test_eval_code_ex test_custom_locals).
         let is_class_body = flags & CO_OPTIMIZED == 0 && co_name != "<module>";
         let mut co = CodeObject {
-            name: co_name,
-            qualname: co_qualname,
+            name: co_name.into(),
+            qualname: co_qualname.into(),
             filename: string_of(&filename, "co_filename")?.into(),
             caches: CacheTable::with_len(decoded.instructions.len()),
             vm_ext: weavepy_compiler::VmExt::default(),

From b9342ccc854f41828a9d8f8b262df5771926ac25 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 10:56:40 -0700
Subject: [PATCH 071/173] perf: hash pooled names eight bytes at a time; leave
 qualnames out

The name pool hashed with FNV-1a, a multiply per byte, and every
co_qualname went through it, though a qualified name (`C.method`) is
seldom shared: importing asyncio made about 20,600 pool lookups at
about 360 instructions each.

The pool now hashes with FxHash's mix, a multiply per eight bytes, and
a qualified name that isn't also a plain name is made outside the pool
(`Name::unpooled`), costing an allocation as a String did.

import asyncio: 463.98M -> 461.56M instructions (min of 8), 27.43 ->
27.19 MB peak RSS.
---
 crates/weavepy-compiler/src/lib.rs         | 10 ++---
 crates/weavepy-compiler/src/name.rs        | 45 +++++++++++++++-------
 crates/weavepy-compiler/src/native_code.rs | 11 +++++-
 3 files changed, 46 insertions(+), 20 deletions(-)

diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs
index aa557135..cde73cb8 100644
--- a/crates/weavepy-compiler/src/lib.rs
+++ b/crates/weavepy-compiler/src/lib.rs
@@ -5878,7 +5878,7 @@ impl Compiler {
                 .insert("__classdict__".to_owned(), Binding::Free);
             inner.free_order.push("__classdict__".to_owned());
         }
-        inner.co.qualname = self.compute_child_qualname(hidden_name).into();
+        inner.co.qualname = Name::unpooled(&self.compute_child_qualname(hidden_name));
         inner.co.arg_count = arg_count;
         inner.co.varnames = hidden_params.iter().map(Name::from).collect();
         inner.current_line = entry_line.unwrap_or(self.current_line);
@@ -6362,7 +6362,7 @@ impl Compiler {
                 .insert("__classdict__".to_owned(), Binding::Free);
             inner.free_order.push("__classdict__".to_owned());
         }
-        inner.co.qualname = self.compute_child_qualname(display).into();
+        inner.co.qualname = Name::unpooled(&self.compute_child_qualname(display));
         inner.co.arg_count = arg_count;
         inner.co.posonly_count = posonly_count;
         inner.co.kwonly_count = kwonly_count;
@@ -6618,7 +6618,7 @@ impl Compiler {
                 .insert("__classdict__".to_owned(), Binding::Free);
             inner.free_order.push("__classdict__".to_owned());
         }
-        inner.co.qualname = self.compute_annotation_child_qualname(name).into();
+        inner.co.qualname = Name::unpooled(&self.compute_annotation_child_qualname(name));
         // A lambda or comprehension inside the annotation scope is
         // named from *this* scope (`compiler_set_qualname` looks
         // through the annotation parent).
@@ -7102,7 +7102,7 @@ impl Compiler {
         // A class body never reports `CO_NESTED` itself (no
         // `CO_OPTIMIZED`), but its methods inherit the nesting.
         inner.co.is_nested = self.child_is_nested();
-        inner.co.qualname = self.compute_child_qualname(name).into();
+        inner.co.qualname = Name::unpooled(&self.compute_child_qualname(name));
         inner.current_line = entry_line.unwrap_or(self.current_line);
         // CPython only gives a class body the `__class__` closure cell —
         // and the trailing `__classcell__` store — when a method actually
@@ -11801,7 +11801,7 @@ impl Compiler {
         // PEP 3155: a comprehension scope gets a dotted qualname like any
         // other nested scope (`C.m.<locals>.<genexpr>`); CPython's
         // `compiler_set_qualname` doesn't special-case comprehensions.
-        inner.co.qualname = self.compute_child_qualname(name).into();
+        inner.co.qualname = Name::unpooled(&self.compute_child_qualname(name));
         inner.co.arg_count = 1;
         inner.co.varnames.push(Name::from(".0".to_owned()));
         inner.bindings.insert(".0".to_owned(), Binding::Local);
diff --git a/crates/weavepy-compiler/src/name.rs b/crates/weavepy-compiler/src/name.rs
index 3a623152..76593248 100644
--- a/crates/weavepy-compiler/src/name.rs
+++ b/crates/weavepy-compiler/src/name.rs
@@ -30,44 +30,63 @@ impl Name {
             .intern(s)
     }
 
+    /// A name equal to `s` outside the pool, for text few code objects
+    /// share (a qualified name): it costs no pool lookup, and equality
+    /// still compares the text.
+    pub fn unpooled(s: &str) -> Name {
+        Name(Arc::from(s))
+    }
+
     #[inline]
     pub fn as_str(&self) -> &str {
         &self.0
     }
 
-    /// Whether `a` and `b` share one allocation (pooled names always do
-    /// when equal).
+    /// Whether `a` and `b` share one allocation (equal pooled names always
+    /// do).
     #[inline]
     pub fn ptr_eq(a: &Name, b: &Name) -> bool {
         Arc::ptr_eq(&a.0, &b.0)
     }
 }
 
-/// The pool's hasher: FNV-1a, which is quick over short identifiers.
-struct Fnv(u64);
+/// The pool's hasher (FxHash's): a rotate, an exclusive-or and a
+/// multiply per eight bytes, which is quick over short identifiers.
+#[derive(Default)]
+struct Fx(u64);
 
-impl Default for Fnv {
-    fn default() -> Self {
-        Fnv(0xcbf2_9ce4_8422_2325)
+impl Fx {
+    #[inline]
+    fn add(&mut self, word: u64) {
+        self.0 = (self.0.rotate_left(5) ^ word).wrapping_mul(0x517c_c1b7_2722_0a95);
     }
 }
 
-impl Hasher for Fnv {
+impl Hasher for Fx {
     fn finish(&self) -> u64 {
         self.0
     }
 
     fn write(&mut self, bytes: &[u8]) {
-        let mut h = self.0;
-        for &b in bytes {
-            h = (h ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3);
+        let mut chunks = bytes.chunks_exact(8);
+        for c in &mut chunks {
+            self.add(u64::from_le_bytes(c.try_into().expect("eight bytes")));
         }
-        self.0 = h;
+        let rest = chunks.remainder();
+        if !rest.is_empty() {
+            let mut last = [0u8; 8];
+            last[..rest.len()].copy_from_slice(rest);
+            self.add(u64::from_le_bytes(last));
+        }
+    }
+
+    fn write_u8(&mut self, b: u8) {
+        self.add(u64::from(b));
     }
 }
 
 struct Pool {
-    names: Option<HashSet<Arc<str>, BuildHasherDefault<Fnv>>>,
+    names: Option<HashSet<Arc<str>, BuildHasherDefault<Fx>>>,
     /// The size past which the next insertion sweeps unused names.
     sweep_at: usize,
 }
diff --git a/crates/weavepy-compiler/src/native_code.rs b/crates/weavepy-compiler/src/native_code.rs
index f2748cee..3eb3cb4e 100644
--- a/crates/weavepy-compiler/src/native_code.rs
+++ b/crates/weavepy-compiler/src/native_code.rs
@@ -459,9 +459,15 @@ impl Reader<'_> {
 
     /// An identifier [`Writer::name`] wrote.
     fn name(&mut self, module: &mut Module) -> Option<Name> {
+        self.name_as(module, Name::new)
+    }
+
+    /// An identifier [`Writer::name`] wrote, made with `make` (pooled or
+    /// not) if this is its first appearance.
+    fn name_as(&mut self, module: &mut Module, make: fn(&str) -> Name) -> Option<Name> {
         Some(match self.uint()? {
             0 => {
-                let name = Name::new(std::str::from_utf8(self.raw()?).ok()?);
+                let name = make(std::str::from_utf8(self.raw()?).ok()?);
                 module.names.push(name.clone());
                 name
             }
@@ -543,7 +549,8 @@ impl Reader<'_> {
             return None;
         }
         let name = self.name(module)?;
-        let qualname = self.name(module)?;
+        // (A qualified name other than a plain name is seldom shared.)
+        let qualname = self.name_as(module, Name::unpooled)?;
         let n = self.len()?;
         let mut instructions = Vec::with_capacity(n);
         for _ in 0..n {

From 96f61105f5e0abfd1e9bc8eef9a9a8714516717c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 11:20:22 -0700
Subject: [PATCH 072/173] perf: keep the code cache's position tables in one
 section

A module's line, column and wire-mark tables were interleaved with its
code in the code cache's native form. Decoding walked each line table to
find its end, copied every table into a growing buffer (about 14,500
appends for asyncio), then copied the buffer once more.

The native form (version 7) now writes every table to a section after
the code, which records only each table's length. The reader copies the
section once and hands each table its range, decoding it only when
something reads it (as before). A default code object's empty names now
share one pooled string instead of looking it up twice.

import asyncio: 458.2M -> 450.7M instructions, import re 128.1M ->
126.1M, import typing, dataclasses, enum 257.2M -> 252.3M.
---
 crates/weavepy-compiler/src/lib.rs         |   4 +
 crates/weavepy-compiler/src/name.rs        |   4 +-
 crates/weavepy-compiler/src/native_code.rs | 136 ++++++++++++---------
 3 files changed, 86 insertions(+), 58 deletions(-)

diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs
index cde73cb8..ad99d52c 100644
--- a/crates/weavepy-compiler/src/lib.rs
+++ b/crates/weavepy-compiler/src/lib.rs
@@ -1082,6 +1082,10 @@ impl Encoded {
     pub(crate) fn new(buffer: std::sync::Arc<TableBuffer>, start: u32, end: u32) -> Self {
         Encoded { buffer, start, end }
     }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.start == self.end
+    }
 }
 
 /// A code object's wire marks (see [`CodeObject::wire_marks`]): a vector,
diff --git a/crates/weavepy-compiler/src/name.rs b/crates/weavepy-compiler/src/name.rs
index 76593248..0ed7a973 100644
--- a/crates/weavepy-compiler/src/name.rs
+++ b/crates/weavepy-compiler/src/name.rs
@@ -113,8 +113,10 @@ impl Pool {
 }
 
 impl Default for Name {
+    /// The empty name, made once (every default code object has two).
     fn default() -> Self {
-        Name::new("")
+        static EMPTY: std::sync::OnceLock<Name> = std::sync::OnceLock::new();
+        EMPTY.get_or_init(|| Name::new("")).clone()
     }
 }
 
diff --git a/crates/weavepy-compiler/src/native_code.rs b/crates/weavepy-compiler/src/native_code.rs
index 3eb3cb4e..be40d8ec 100644
--- a/crates/weavepy-compiler/src/native_code.rs
+++ b/crates/weavepy-compiler/src/native_code.rs
@@ -11,6 +11,12 @@
 //! the module's filename, which the reader supplies. A module's name
 //! tables refer to its distinct identifiers by number after their first
 //! appearance, so the reader decodes and pools each identifier once.
+//!
+//! The layout: the version byte, the offset of the table section as four
+//! little-endian bytes, the module's code object, then the table section,
+//! which holds every code object's line, column and wire-mark tables in
+//! the order the code records their lengths. A reader copies the section
+//! whole and decodes a table only when something reads it.
 
 use std::sync::Arc;
 
@@ -18,15 +24,21 @@ use crate::bytecode::{CacheTable, Instruction, OpCode};
 use crate::{CodeObject, ColSpan, Constant, ExcHandler, Name};
 
 /// The layout revision; bump it whenever the encoding changes.
-pub const VERSION: u8 = 6;
+pub const VERSION: u8 = 7;
 
 /// Encode `code` (and its nested code objects). `None` for a code object
 /// the format doesn't carry: one with raw CPython wire overrides, or with
 /// a constant that has no fixed value.
 pub fn encode(code: &CodeObject) -> Option<Vec<u8>> {
-    let mut w = Writer(Vec::with_capacity(4096), std::collections::HashMap::new());
+    let mut w = Writer::new(Vec::with_capacity(4096));
     w.byte(VERSION);
+    // The table section's offset, filled in below.
+    w.0.extend_from_slice(&[0; 4]);
     w.code(code)?;
+    let at = u32::try_from(w.0.len()).ok()?;
+    w.0[1..5].copy_from_slice(&at.to_le_bytes());
+    let tables = std::mem::take(&mut w.2);
+    w.0.extend_from_slice(&tables);
     Some(w.0)
 }
 
@@ -97,54 +109,71 @@ pub(crate) fn decode_wire_marks(bytes: &[u8]) -> Option<Vec<u8>> {
 /// holds all their line and column tables and wire marks (see
 /// `LineTable`).
 pub fn decode(bytes: &[u8], filename: &str) -> Option<CodeObject> {
-    let mut r = Reader { bytes, pos: 0 };
-    if r.byte()? != VERSION {
+    if bytes.first() != Some(&VERSION) {
         return None;
     }
+    let at = u32::from_le_bytes(bytes.get(1..5)?.try_into().ok()?) as usize;
+    let code_bytes = bytes.get(5..at)?;
+    let tables = &bytes[at..];
+    let mut r = Reader {
+        bytes: code_bytes,
+        pos: 0,
+    };
     let mut module = Module {
         filename: Arc::from(filename),
-        tables: Vec::new(),
-        buffer: Arc::new(std::sync::OnceLock::new()),
+        buffer: Arc::new(std::sync::OnceLock::from(Box::from(tables))),
+        tables_len: tables.len(),
+        next: 0,
         names: Vec::new(),
     };
     let code = r.code(&mut module, 0)?;
-    if r.pos != bytes.len() {
-        return None;
-    }
-    // (Copied rather than shrunk in place, which can keep the vector's
-    // spare capacity.)
-    let _ = module.buffer.set(Box::from(module.tables.as_slice()));
-    Some(code)
+    (r.pos == code_bytes.len() && module.next == tables.len()).then_some(code)
 }
 
 /// What the code objects of one module decoded by [`decode`] share.
 struct Module {
     filename: Arc<str>,
-    /// Every code object's encoded line and column tables and wire
-    /// marks, in order.
-    tables: Vec<u8>,
-    /// Where `tables` goes once the decoding is done.
-    buffer: Arc<std::sync::OnceLock<Box<[u8]>>>,
+    /// The table section (see the module docs).
+    buffer: Arc<crate::TableBuffer>,
+    /// The section's length, and where its next table starts.
+    tables_len: usize,
+    next: usize,
     /// The module's identifiers so far, in order of first appearance
     /// (see [`Writer::names`]).
     names: Vec<Name>,
 }
 
 impl Module {
-    /// Append an encoded table to the shared buffer.
-    fn table(&mut self, bytes: &[u8]) -> Option<crate::Encoded> {
-        let start = u32::try_from(self.tables.len()).ok()?;
-        self.tables.extend_from_slice(bytes);
-        let end = u32::try_from(self.tables.len()).ok()?;
-        Some(crate::Encoded::new(self.buffer.clone(), start, end))
+    /// The table section's next `len` bytes.
+    fn table(&mut self, len: usize) -> Option<crate::Encoded> {
+        let start = self.next;
+        let end = start
+            .checked_add(len)
+            .filter(|&end| end <= self.tables_len)?;
+        self.next = end;
+        Some(crate::Encoded::new(
+            self.buffer.clone(),
+            u32::try_from(start).ok()?,
+            u32::try_from(end).ok()?,
+        ))
     }
 }
 
-/// The output, and the number of each identifier written so far (see
-/// [`Writer::names`]).
-struct Writer(Vec<u8>, std::collections::HashMap<String, u32>);
+/// The output, the number of each identifier written so far (see
+/// [`Writer::names`]), and the table section (see the module docs).
+struct Writer(Vec<u8>, std::collections::HashMap<String, u32>, Vec<u8>);
 
 impl Writer {
+    fn new(out: Vec<u8>) -> Self {
+        Writer(out, std::collections::HashMap::new(), Vec::new())
+    }
+
+    /// Move a table into the table section, recording its length.
+    fn table(&mut self, table: Vec<u8>) {
+        self.uint(table.len() as u64);
+        self.2.extend_from_slice(&table);
+    }
+
     fn byte(&mut self, b: u8) {
         self.0.push(b);
     }
@@ -276,12 +305,12 @@ impl Writer {
             self.byte(u8::from(h.push_lasti));
         }
         // Line numbers in runs: a statement's instructions share a line.
-        self.line_runs(&c.linetable);
-        // The column spans as one length-prefixed block, which a reader
-        // keeps encoded until something reads a column (see `ColTable`).
-        let mut cols = Writer(Vec::new(), std::collections::HashMap::new());
+        let mut lines = Writer::new(Vec::new());
+        lines.line_runs(&c.linetable);
+        self.table(lines.0);
+        let mut cols = Writer::new(Vec::new());
         cols.col_runs(&c.coltable)?;
-        self.bytes(&cols.0);
+        self.table(cols.0);
         self.uint(u64::from(c.arg_count));
         self.uint(u64::from(c.posonly_count));
         self.uint(u64::from(c.kwonly_count));
@@ -306,7 +335,12 @@ impl Writer {
         self.uint(u64::from(c.future_flags));
         self.uint(c.stacksize.map_or(0, |s| u64::from(s) + 1));
         self.u32s(c.no_interrupt_jumps());
-        self.wire_marks(&c.wire_marks)?;
+        // (No table at all for code without marks.)
+        let mut marks = Writer::new(Vec::new());
+        if !c.wire_marks.is_empty() {
+            marks.wire_marks(&c.wire_marks)?;
+        }
+        self.table(marks.0);
         self.strs(c.hidden_locals());
         self.strs(c.const_identifiers());
         Some(())
@@ -517,25 +551,11 @@ impl Reader<'_> {
         Some(())
     }
 
-    /// The bytes of a line-number block, checked as decoding would but
-    /// left encoded (see `LineTable`).
-    fn line_runs_raw(&mut self) -> Option<&[u8]> {
-        let start = self.pos;
-        self.line_runs(|_, _| {})?;
-        Some(&self.bytes[start..self.pos])
-    }
-
-    /// The bytes of a wire-mark block, checked for length but left
-    /// encoded (see `WireMarks`); empty for code without marks.
-    fn wire_marks_raw(&mut self) -> Option<&[u8]> {
-        let start = self.pos;
-        let n = self.table_len()?;
-        let end = self.pos.checked_add(n.div_ceil(2))?;
-        if end > self.bytes.len() {
-            return None;
-        }
-        self.pos = end;
-        Some(if n == 0 { &[] } else { &self.bytes[start..end] })
+    /// The next table of the module's table section (see the module
+    /// docs), left encoded.
+    fn table(&mut self, module: &mut Module) -> Option<crate::Encoded> {
+        let len = usize::try_from(self.uint()?).ok()?;
+        module.table(len)
     }
 
     fn f64(&mut self) -> Option<f64> {
@@ -577,8 +597,8 @@ impl Reader<'_> {
                 push_lasti: self.byte()? != 0,
             });
         }
-        let linetable = crate::LineTable::encoded(module.table(self.line_runs_raw()?)?);
-        let coltable = crate::ColTable::encoded(module.table(self.raw()?)?);
+        let linetable = crate::LineTable::encoded(self.table(module)?);
+        let coltable = crate::ColTable::encoded(self.table(module)?);
         let arg_count = self.u32()?;
         let posonly_count = self.u32()?;
         let kwonly_count = self.u32()?;
@@ -587,9 +607,11 @@ impl Reader<'_> {
         let future_flags = self.u32()?;
         let stacksize = self.u32()?.checked_sub(1);
         let no_interrupt_jumps = self.u32s()?;
-        let wire_marks = match self.wire_marks_raw()? {
-            [] => crate::WireMarks::default(),
-            marks => crate::WireMarks::encoded(module.table(marks)?),
+        let marks = self.table(module)?;
+        let wire_marks = if marks.is_empty() {
+            crate::WireMarks::default()
+        } else {
+            crate::WireMarks::encoded(marks)
         };
         let mut rare = crate::CodeRare::default();
         rare.set(crate::RareFields {

From 9abf0218990a03bc587ebec15940e41400a25ec8 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 11:49:58 -0700
Subject: [PATCH 073/173] perf: map large table sections of cached modules
 instead of reading them

A cached module's table section (its line, column and wire-mark tables)
was read into the heap with the rest of the artifact, though most of its
tables are never read: column spans only by tracebacks and
co_positions(), wire marks only by the CPython wire form. Importing
asyncio held about 0.8 MB of them.

The disk layer now reads the artifact's code and maps its table section
when the section is 8 KB or longer (46 of the 132 modules the import
probes load, with 1.0 of their 1.3 MB of tables). Pages of the mapping
are faulted in only for the tables something reads. The cache's writers
replace an artifact by renaming a new file over it, never in place, so
the mapping keeps the contents just validated. `TableBuffer` holds
either owned bytes or such a mapping, and `native_code::decode_split`
decodes code read apart from its tables.

Peak RSS: import asyncio 27.13 -> 26.40 MB, import typing, dataclasses,
enum 21.20 -> 20.67 MB, import pathlib, datetime, urllib.parse,
statistics 23.11 -> 22.68 MB, import re 15.00 -> 14.83 MB; instructions
unchanged.
---
 crates/weavepy-compiler/src/lib.rs         | 28 +++++++++---
 crates/weavepy-compiler/src/native_code.rs | 32 +++++++++++---
 crates/weavepy-vm/src/frozen_code_cache.rs | 50 +++++++++++++++++++---
 3 files changed, 92 insertions(+), 18 deletions(-)

diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs
index ad99d52c..48a1dac8 100644
--- a/crates/weavepy-compiler/src/lib.rs
+++ b/crates/weavepy-compiler/src/lib.rs
@@ -844,9 +844,26 @@ struct LazyTable<T> {
     _items: std::marker::PhantomData<Vec<T>>,
 }
 
-/// A module's encoded position tables, filled once the decoding is done
-/// (see `native_code::decode`).
-pub(crate) type TableBuffer = std::sync::OnceLock<Box<[u8]>>;
+/// The table section of a module in the code cache's native form (see
+/// `native_code`): its own bytes, or, say, a mapping of the cache file
+/// that only the tables something reads ever page in.
+pub struct TableBuffer(Box<dyn AsRef<[u8]> + Send + Sync>);
+
+impl TableBuffer {
+    pub fn new(bytes: impl AsRef<[u8]> + Send + Sync + 'static) -> Self {
+        TableBuffer(Box::new(bytes))
+    }
+
+    fn bytes(&self) -> &[u8] {
+        (*self.0).as_ref()
+    }
+}
+
+impl std::fmt::Debug for TableBuffer {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "TableBuffer({} bytes)", self.bytes().len())
+    }
+}
 
 /// The tag of an encoded [`LazyTable`]'s state.
 const LAZY_ENCODED: usize = 1;
@@ -916,8 +933,8 @@ impl<T> LazyTable<T> {
             // published table keeps alive for any racing reader).
             let source = unsafe { std::sync::Arc::from_raw(buffer) };
             let items = source
-                .get()
-                .and_then(|b| b.get(self.start as usize..self.end as usize))
+                .bytes()
+                .get(self.start as usize..self.end as usize)
                 .and_then(decode)
                 .unwrap_or_default();
             (items, Some(source))
@@ -1072,7 +1089,6 @@ impl LineTable {
 /// module shares (see `native_code::decode`), so a module's tables take
 /// one allocation rather than three per code object.
 pub(crate) struct Encoded {
-    /// Set once the module's decoding finishes, before any table is read.
     buffer: std::sync::Arc<TableBuffer>,
     start: u32,
     end: u32,
diff --git a/crates/weavepy-compiler/src/native_code.rs b/crates/weavepy-compiler/src/native_code.rs
index be40d8ec..349e545d 100644
--- a/crates/weavepy-compiler/src/native_code.rs
+++ b/crates/weavepy-compiler/src/native_code.rs
@@ -109,25 +109,43 @@ pub(crate) fn decode_wire_marks(bytes: &[u8]) -> Option<Vec<u8>> {
 /// holds all their line and column tables and wire marks (see
 /// `LineTable`).
 pub fn decode(bytes: &[u8], filename: &str) -> Option<CodeObject> {
+    let at = tables_at(bytes)?;
+    let tables = crate::TableBuffer::new(Box::<[u8]>::from(bytes.get(at..)?));
+    decode_split(&bytes[..at], tables, filename)
+}
+
+/// Where the table section of what [`encode`] wrote starts, from its
+/// first five bytes (or more); `None` for input it didn't write.
+pub fn tables_at(bytes: &[u8]) -> Option<usize> {
     if bytes.first() != Some(&VERSION) {
         return None;
     }
     let at = u32::from_le_bytes(bytes.get(1..5)?.try_into().ok()?) as usize;
-    let code_bytes = bytes.get(5..at)?;
-    let tables = &bytes[at..];
+    (at >= 5).then_some(at)
+}
+
+/// [`decode`] for input split at [`tables_at`]: `code` is what comes
+/// before the table section and `tables` holds the section, which the
+/// decoded code objects keep, reading a table from it only when
+/// something asks for that table.
+pub fn decode_split(code: &[u8], tables: crate::TableBuffer, filename: &str) -> Option<CodeObject> {
+    if tables_at(code)? != code.len() {
+        return None;
+    }
+    let tables_len = tables.bytes().len();
     let mut r = Reader {
-        bytes: code_bytes,
+        bytes: &code[5..],
         pos: 0,
     };
     let mut module = Module {
         filename: Arc::from(filename),
-        buffer: Arc::new(std::sync::OnceLock::from(Box::from(tables))),
-        tables_len: tables.len(),
+        buffer: Arc::new(tables),
+        tables_len,
         next: 0,
         names: Vec::new(),
     };
-    let code = r.code(&mut module, 0)?;
-    (r.pos == code_bytes.len() && module.next == tables.len()).then_some(code)
+    let decoded = r.code(&mut module, 0)?;
+    (r.pos == r.bytes.len() && module.next == tables_len).then_some(decoded)
 }
 
 /// What the code objects of one module decoded by [`decode`] share.
diff --git a/crates/weavepy-vm/src/frozen_code_cache.rs b/crates/weavepy-vm/src/frozen_code_cache.rs
index 9a0a6ebc..7fe147bc 100644
--- a/crates/weavepy-vm/src/frozen_code_cache.rs
+++ b/crates/weavepy-vm/src/frozen_code_cache.rs
@@ -162,12 +162,19 @@ fn disk_path(name: &str) -> Option<PathBuf> {
 /// path may differ from the writer's), installed into the in-memory
 /// cache, and returned. Any mismatch or decode failure is a miss.
 pub fn get_disk(name: &str, source: &str, filename: &str) -> Option<CodeObject> {
-    let bytes = std::fs::read(disk_path(name)?).ok()?;
-    if bytes.len() < FROZEN_HEADER_LEN || &bytes[0..4] != FROZEN_MAGIC {
+    use std::io::Read;
+    use weavepy_compiler::native_code;
+    let mut file = std::fs::File::open(disk_path(name)?).ok()?;
+    let size = usize::try_from(file.metadata().ok()?.len()).ok()?;
+    // The header and the payload's first bytes, which say where its table
+    // section starts.
+    let mut head = [0u8; FROZEN_HEADER_LEN + 5];
+    file.read_exact(&mut head).ok()?;
+    if &head[0..4] != FROZEN_MAGIC {
         return None;
     }
-    let len = u32::from_le_bytes(bytes[8..12].try_into().ok()?);
-    let hash = u64::from_le_bytes(bytes[12..20].try_into().ok()?);
+    let len = u32::from_le_bytes(head[8..12].try_into().ok()?);
+    let hash = u64::from_le_bytes(head[12..20].try_into().ok()?);
     // An embedded module's hash comes from the build: computing it here
     // would page in the whole source text the cached code stands for.
     if len as usize != source.len() || hash != crate::stdlib::frozen_source_hash(name, source) {
@@ -177,9 +184,42 @@ pub fn get_disk(name: &str, source: &str, filename: &str) -> Option<CodeObject>
     // same artifact again, and a resident clone of every loaded module's
     // code would double its footprint in the (usual) single-interpreter
     // process.
-    weavepy_compiler::native_code::decode(&bytes[FROZEN_HEADER_LEN..], filename)
+    let at = native_code::tables_at(&head[FROZEN_HEADER_LEN..])?;
+    let tables_start = FROZEN_HEADER_LEN.checked_add(at).filter(|&t| t <= size)?;
+    let mut payload = Vec::with_capacity(size - FROZEN_HEADER_LEN);
+    payload.extend_from_slice(&head[FROZEN_HEADER_LEN..]);
+    if size - tables_start < MAP_TABLES_FROM {
+        file.read_to_end(&mut payload).ok()?;
+        return native_code::decode(&payload, filename);
+    }
+    // A large table section is mapped rather than read: most of its
+    // tables (column spans, wire marks) are never read, and pages never
+    // touched take no memory.
+    (&mut file)
+        .take((at - payload.len()) as u64)
+        .read_to_end(&mut payload)
+        .ok()?;
+    // SAFETY: the cache's writers replace an artifact by renaming a new
+    // file over it, never by changing it in place, so the mapped bytes
+    // keep the contents just validated (a replaced or removed file keeps
+    // them while mapped).
+    let tables = unsafe {
+        memmap2::MmapOptions::new()
+            .offset(tables_start as u64)
+            .len(size - tables_start)
+            .map(&file)
+    }
+    .ok()?;
+    native_code::decode_split(
+        &payload,
+        weavepy_compiler::TableBuffer::new(tables),
+        filename,
+    )
 }
 
+/// The shortest table section [`get_disk`] maps rather than reads.
+const MAP_TABLES_FROM: usize = 8 * 1024;
+
 /// Persist a freshly-compiled frozen module to the disk cache.
 /// Best-effort: any I/O or marshal failure is silently ignored (a
 /// read-only cache dir must not fail the import).

From fd76b9a1a6a4881f71b722ca08498b6d03985f37 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 12:55:53 -0700
Subject: [PATCH 074/173] perf: share the frame compiler's exit tails per
 status

Every exit of a compiled frame wrote the depth, the next pc and the last
instruction to the state with stores of constants, then returned its
status: six to eight IR instructions, repeated at each of a large body's
hundreds of exits (tomllib's parse_value has 331), which made up about
a third of its IR.

An exit now passes the three values to a tail block that every exit
with the same status shares, which does the stores and the return. The
lean threshold still measures a body as if its tails were in line, so
the bodies lowered lean don't change.

tomllib's parse_value: 6,537 -> 5,646 IR instructions and 42,432 ->
33,360 bytes of code; loads: 6,152 -> 5,205 and 39,304 -> 29,680. Peak
heap over the suite (exact, from alloc-profile builds): geomean 0.9935,
-2 to -3% on nqueens, mini_interpreter, sudoku, tomllib_loads and
unpack_sequence. Suite instructions unchanged.
---
 crates/weavepy-vm/src/frame_jit.rs | 68 +++++++++++++++++++++++++++---
 1 file changed, 62 insertions(+), 6 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 2e27f1e6..09f72b33 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -5885,11 +5885,14 @@ fn compile_with(
         direct = std::mem::take(&mut lower.direct_sites);
         held = std::mem::take(&mut lower.held);
         mods = std::mem::take(&mut lower.mod_caches);
+        // (Measured as if every exit's tail were in line, the size the
+        // threshold was tuned with.)
+        let size_in_line = lower.tail_savings;
         if done {
             lower.b.seal_all_blocks();
             lower.b.finalize();
         }
-        if done && !lean && engine.ctx.func.dfg.num_insts() > lean_above() {
+        if done && !lean && engine.ctx.func.dfg.num_insts() + size_in_line > lean_above() {
             lean = true;
             engine.fbctx = FunctionBuilderContext::new();
             engine.module.clear_context(&mut engine.ctx);
@@ -6046,6 +6049,13 @@ struct Lower<'a> {
     /// Side exits waiting for their blocks to be filled (after the body:
     /// the builder fills one block at a time).
     side_exits: Vec<SideExit>,
+    /// The blocks that end the exits returning a status (see
+    /// [`Self::exit`]), by status and whether they store the last
+    /// instruction; filled after the side exits.
+    exit_tails: Vec<(u32, bool, Block)>,
+    /// The IR instructions the shared tails saved over writing each
+    /// exit's state and return in line.
+    tail_savings: usize,
     /// The `LOAD_GLOBAL` and `LOAD_ATTR` sites' caches, which the code
     /// addresses.
     #[allow(clippy::vec_box)]
@@ -6231,6 +6241,8 @@ impl<'a> Lower<'a> {
             depth: 0,
             sigs: Vec::new(),
             side_exits: Vec::new(),
+            exit_tails: Vec::new(),
+            tail_savings: 0,
             global_caches: Vec::new(),
             slot_caches: Vec::new(),
             method_caches: Vec::new(),
@@ -6503,17 +6515,60 @@ impl<'a> Lower<'a> {
     }
 
     /// Return `status` with the virtual stack written out and `pc` next.
+    /// The state's stores and the return are a tail block every exit with
+    /// the same status shares, which takes the depth, `pc` and the last
+    /// instruction as arguments: a body has hundreds of exits, and their
+    /// tails would otherwise be most of its IR.
     fn exit(&mut self, status: u32, pc: usize) {
         let cold = std::mem::replace(&mut self.cold, true);
         self.flush();
         self.cold = cold;
+        let last = self.last_pc;
+        let tail = match self
+            .exit_tails
+            .iter()
+            .find(|&&(s, l, _)| s == status && l == last.is_some())
+        {
+            Some(&(_, _, tail)) => tail,
+            None => {
+                let tail = self.b.create_block();
+                let params = if last.is_some() { 3 } else { 2 };
+                for _ in 0..params {
+                    self.b.append_block_param(tail, types::I64);
+                }
+                self.exit_tails.push((status, last.is_some(), tail));
+                tail
+            }
+        };
+        // In line, the status constant, the stores and the return less
+        // the jump.
+        self.tail_savings += if last.is_some() { 4 } else { 3 };
         let l = self.b.ins().iconst(types::I64, self.depth as i64);
-        self.b.ins().store(FLAGS, l, self.st, S_LEN);
-        self.store_last();
         let pcv = self.b.ins().iconst(types::I64, pc as i64);
-        self.b.ins().store(FLAGS, pcv, self.st, S_PC);
-        let s = self.b.ins().iconst(types::I32, i64::from(status));
-        self.b.ins().return_(&[s]);
+        match last {
+            Some(last) => {
+                let lv = self.b.ins().iconst(types::I64, last as i64);
+                self.b.ins().jump(tail, &[l.into(), pcv.into(), lv.into()]);
+            }
+            None => {
+                self.b.ins().jump(tail, &[l.into(), pcv.into()]);
+            }
+        }
+    }
+
+    /// Fill the exit tails (see [`Self::exit`]).
+    fn exit_tails(&mut self) {
+        for (status, _, tail) in std::mem::take(&mut self.exit_tails) {
+            self.b.switch_to_block(tail);
+            let params = self.b.block_params(tail).to_vec();
+            self.b.ins().store(FLAGS, params[0], self.st, S_LEN);
+            self.b.ins().store(FLAGS, params[1], self.st, S_PC);
+            if let Some(&last) = params.get(2) {
+                self.b.ins().store(FLAGS, last, self.st, S_LAST);
+            }
+            let s = self.b.ins().iconst(types::I32, i64::from(status));
+            self.b.ins().return_(&[s]);
+        }
     }
 
     /// Leave for the core loop at `pc` (the instruction about to run),
@@ -6567,6 +6622,7 @@ impl<'a> Lower<'a> {
             }
             self.exit(x.status, x.pc);
         }
+        self.exit_tails();
     }
 
     /// Stop for a queued finalizer (with `pc` next), from the current

From da46e7c795e5d25672d2fd09259e09412fd82c7b Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 12:56:05 -0700
Subject: [PATCH 075/173] perf: remap the stale linker order file onto current
 symbols

The checked-in order file named its functions with crate hashes from an
older toolchain, so only `_main` matched and the linker laid the binary
out with no ordering: start-up touched code pages spread over the whole
text segment.

scripts/gen-order-file.sh can't regenerate it today (its instrumented
binary segfaults during start-up, even for --version, with rustc 1.99),
so this remaps the existing order onto the current symbols with
scripts/remap-order-file.py's matching (crate hashes paired by crate
name, then demangled names): 1,984 exact and 636 demangled matches for
the release profile's names. The file also carries the quick profile's
names (2,192 exact, 577 demangled), whose crate hashes differ, so the
quick builds used for performance work get the same layout; the linker
ignores the names a build doesn't define.

Quick build, peak RSS: -c pass 10.89 -> 10.18 MB, import asyncio 26.46
-> 25.90 MB; chaos 30.13 -> 29.37, deepcopy 31.21 -> 30.41, go 28.95
-> 28.30, json_roundtrip 20.82 -> 20.04, sudoku 23.73 -> 22.91 MB.
---
 crates/weavepy-cli/weavepy.order | 8680 +++++++++++++++++-------------
 1 file changed, 5049 insertions(+), 3631 deletions(-)

diff --git a/crates/weavepy-cli/weavepy.order b/crates/weavepy-cli/weavepy.order
index 2d1622f8..a099563c 100644
--- a/crates/weavepy-cli/weavepy.order
+++ b/crates/weavepy-cli/weavepy.order
@@ -1,3632 +1,5050 @@
-__RNvNtCs2D882usVKTN_10weavepy_vm9proc_init16snapshot_std_fds
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm9proc_init16snapshot_std_fds
 _main
-__RNvCs1njKG4L9aB3_7___rustc12___rust_alloc
-__RNvCsdp43gqXvMwI_11weavepy_cli8cli_main
-__RNvNtCs2D882usVKTN_10weavepy_vm9proc_init23restore_initial_std_fds
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod34block_async_signals_current_thread
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod12StoredSigsetE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEBZ_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod12StoredSigsetE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B1M_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner15try_allocate_inCsdp43gqXvMwI_11weavepy_cli
-__RNvCs1njKG4L9aB3_7___rustc14___rust_realloc
-__RNvCs1njKG4L9aB3_7___rustc14___rust_dealloc
-__RNSNvYNCINvNtNtCs82bWklYMk3w_3std6thread9lifecycle15spawn_uncheckedNCNvCsdp43gqXvMwI_11weavepy_cli18run_on_large_stack0lEs_0INtNtNtCs8Mbv00yxnRz_4core3ops8function6FnOnceuE9call_once6vtableB17_
-__RINvNtNtCs82bWklYMk3w_3std3sys9backtrace28___rust_begin_short_backtraceNCNCNCINvNtNtB6_6thread9lifecycle15spawn_uncheckedNCNvCsdp43gqXvMwI_11weavepy_cli18run_on_large_stack0lEs_000uEB1Z_
-__RINvNtNtCs82bWklYMk3w_3std3sys9backtrace28___rust_begin_short_backtraceNCNvCsdp43gqXvMwI_11weavepy_cli18run_on_large_stack0lEB1b_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod36unblock_async_signals_current_thread
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod28install_startup_dispositions
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtNtBa_6poison5mutex5MutexINtNtNtNtBc_11collections4hash3map7HashMaplNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEE10initializeNCINvB1a_11get_or_initNCNvNtNtB2U_6stdlib10signal_mod8handlers0E0zE0E0B2U_
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs8Mbv00yxnRz_4core4cell4CellTyyEEE4withNCNvMNtNtBa_4hash6randomNtB1I_11RandomState3new0B21_ECs2D882usVKTN_10weavepy_vm
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvNtNtB1K_6stdlib10signal_mod23default_int_handler_obj0E0zE0E0B1K_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod1b
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMaplNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertBS_
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRlECs2D882usVKTN_10weavepy_vm
-__RNvXs3_NtNtCs8Mbv00yxnRz_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTlNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherlBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBV_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs3fRkGY2FHAU_9hashbrown10scopeguard10ScopeGuardNtNtBG_3raw13RawTableInnerNCINvMsa_B1u_B1s_14prepare_resizeNtNtCshxvaOLs88l5_5alloc5alloc6GlobalE0EECs2D882usVKTN_10weavepy_vm
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod11set_handler
-__RINvMNtNtNtCs82bWklYMk3w_3std3sys4sync8once_boxINtB3_7OnceBoxNtNtNtNtNtB7_3pal4unix4sync5mutex5MutexE10initializeNCNvMNtNtB5_5mutex7pthreadNtB1S_5Mutex3get0ECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod18set_os_disposition
-__RNvCsdp43gqXvMwI_11weavepy_cli13main_dispatch
-__RNvMNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env7builderNtB2_7Builder12try_from_env
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env7builder7BuilderECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCshxvaOLs88l5_5alloc6string6StringECsdp43gqXvMwI_11weavepy_cli
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE10initializeNCINvB2_11get_or_initNCNvNtNtBX_6stdlib10signal_mod23default_int_handler_obj0E0zEBX_
-__RNvMNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env7builderNtB2_7Builder22with_default_directive
-__RNvMNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env9directiveNtB2_9Directive5parse
-__RNvXse_NtCsc0dy1eKBkNK_12tracing_core8metadataNtB5_11LevelFilterNtNtNtCs8Mbv00yxnRz_4core3str6traits7FromStr8from_str
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env9directive9DirectiveECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env12FromEnvErrorECsdp43gqXvMwI_11weavepy_cli
-__RNvXNtNtCsaTNomOSavyo_18tracing_subscriber8registry7shardedNtB2_8RegistryNtNtCs8Mbv00yxnRz_4core7default7Default7default
-__RNvXsb_NtNtNtCs82bWklYMk3w_3std4sync6poison6rwlockINtB5_6RwLockINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCsc0dy1eKBkNK_12tracing_core10dispatcher9RegistrarEENtNtCs8Mbv00yxnRz_4core7default7Default7defaultB1A_
-__RNvCsdp43gqXvMwI_11weavepy_cli12bridged_args
-__RINvMNtNtNtCs82bWklYMk3w_3std3sys4sync8once_boxINtB3_7OnceBoxNtNtNtNtNtB7_3pal4unix4sync5mutex5MutexE10initializeNCNvMNtNtB5_5mutex7pthreadNtB1S_5Mutex3get0ECsdp43gqXvMwI_11weavepy_cli
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env9directive9DirectiveE8grow_oneBU_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsaTNomOSavyo_18tracing_subscriber
-__RNvMNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env9directiveNtB2_9Directive9to_static
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env5field5MatchEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env9directive9DirectiveEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtNtCsaTNomOSavyo_18tracing_subscriber6filter3env5field5MatchEECsdp43gqXvMwI_11weavepy_cli
-__RNvNtCsaTNomOSavyo_18tracing_subscriber3fmt3fmt
-__RNvNtCsc0dy1eKBkNK_12tracing_core8callsite17register_dispatch
-__RINvMs2_NtCs10qptT6X9kT_9once_cell3impINtB6_8OnceCellINtNtNtNtCs82bWklYMk3w_3std4sync6poison6rwlock6RwLockINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCsc0dy1eKBkNK_12tracing_core10dispatcher9RegistrarEEE10initializeNCINvMs4_NtB8_4syncINtB3t_8OnceCellBP_E11get_or_initNCNvMs9_B3t_INtB3t_4LazyBP_E5force0E0NtNvMs4_B3t_IB3E_pE11get_or_init4VoidEB2h_
-__RNvNtCs10qptT6X9kT_9once_cell3imp18initialize_or_wait
-__RNCINvMs2_NtCs10qptT6X9kT_9once_cell3impINtB8_8OnceCellINtNtNtNtCs82bWklYMk3w_3std4sync6poison6rwlock6RwLockINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCsc0dy1eKBkNK_12tracing_core10dispatcher9RegistrarEEE10initializeNCINvMs4_NtBa_4syncINtB3v_8OnceCellBR_E11get_or_initNCNvMs9_B3v_INtB3v_4LazyBR_E5force0E0NtNvMs4_B3v_IB3G_pE11get_or_init4VoidE0B2j_
-__RNvXs3_NtCs10qptT6X9kT_9once_cell3impNtB5_5GuardNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCsc0dy1eKBkNK_12tracing_core10dispatcher9RegistrarE8grow_oneBQ_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner14grow_amortizedCsc0dy1eKBkNK_12tracing_core
-__RNvYINtNtCsaTNomOSavyo_18tracing_subscriber3fmt10SubscriberNtNtB5_6format13DefaultFieldsNtBX_6FormatNtNtNtB7_6filter3env9EnvFilterENtNtCsc0dy1eKBkNK_12tracing_core10subscriber10Subscriber20on_register_dispatchCsdp43gqXvMwI_11weavepy_cli
-__RNvXs0_NtCsaTNomOSavyo_18tracing_subscriber3fmtINtB5_10SubscriberNtNtB5_6format13DefaultFieldsNtB13_6FormatNtNtNtB7_6filter3env9EnvFilterENtNtCsc0dy1eKBkNK_12tracing_core10subscriber10Subscriber14max_level_hintCsdp43gqXvMwI_11weavepy_cli
-__RNvMs1_NtCsa0A2j3aljHc_11tracing_log10log_tracerNtB5_7Builder4init
-__RNvCs8FydZtWCZBm_3log16set_boxed_logger
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtB6_6string6StringEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapNtNtCs82bWklYMk3w_3std3env6ArgsOsNCNvCs2D882usVKTN_10weavepy_vm15os_args_bridged0EE9from_iterB3g_
-__RNvNvCs2D882usVKTN_10weavepy_vm15os_args_bridged21register_if_ambiguous
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner15try_allocate_inCs2D882usVKTN_10weavepy_vm
-__RNvCsdp43gqXvMwI_11weavepy_cli9real_main
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec14spec_from_iterINtB6_3VecNtNtB8_6string6StringEINtB4_12SpecFromIterBW_INtNtB6_9into_iter8IntoIterBW_EE9from_iterCsdp43gqXvMwI_11weavepy_cli
-__RNvXs0_NtNtCshxvaOLs88l5_5alloc3vec11spec_extendINtB7_3VecNtNtB9_6string6StringEINtB5_10SpecExtendBU_INtNtB7_9into_iter8IntoIterBU_EE11spec_extendCsdp43gqXvMwI_11weavepy_cli
-__RNvXs3_Csdp43gqXvMwI_11weavepy_cliNtB5_3CliNtNtCs8Z8q0vUHYys_12clap_builder6derive4Args12augment_args
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs8Z8q0vUHYys_12clap_builder4util2id2IdE8grow_oneBS_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs8Z8q0vUHYys_12clap_builder
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs8Z8q0vUHYys_12clap_builder7builder9arg_group8ArgGroupE8grow_oneBS_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtCs8Z8q0vUHYys_12clap_builder7builder12value_parser11ValueParserEECsdp43gqXvMwI_11weavepy_cli
-__RINvNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs8Z8q0vUHYys_12clap_builder
-__RNvMs4_NtCs8Z8q0vUHYys_12clap_builder7mkeymapNtB5_7MKeyMap4push
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs8Z8q0vUHYys_12clap_builder7builder3arg3ArgE8grow_oneBS_
-__RINvMs_NtNtCs8Z8q0vUHYys_12clap_builder7builder3argNtB5_3Arg12value_parserINtNtB7_12value_parser20RangedI64ValueParserhEECsdp43gqXvMwI_11weavepy_cli
-__RINvMs0_NtNtCs8Z8q0vUHYys_12clap_builder7builder7commandNtB6_7Command5aboutReECsdp43gqXvMwI_11weavepy_cli
-__RINvMs0_NtNtCs8Z8q0vUHYys_12clap_builder7builder7commandNtB6_7Command10long_aboutReECsdp43gqXvMwI_11weavepy_cli
-__RINvMs0_NtNtCs8Z8q0vUHYys_12clap_builder7builder7commandNtB6_7Command8bin_nameReECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCshxvaOLs88l5_5alloc6string6StringECsdp43gqXvMwI_11weavepy_cli
-__RINvMNtNtCs8Z8q0vUHYys_12clap_builder7builder7commandNtB3_7Command16get_matches_fromINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB1p_6string6StringEB1R_ECsdp43gqXvMwI_11weavepy_cli
-__RNvMs3_NtNtCs8Z8q0vUHYys_12clap_builder7builder7commandNtB5_7Command9__do_parse
-__RNvMs3_NtNtCs8Z8q0vUHYys_12clap_builder7builder7commandNtB5_7Command11__build_self
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs8Z8q0vUHYys_12clap_builder7builder7command7CommandEEB1e_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs8Z8q0vUHYys_12clap_builder7mkeymap3KeyE8grow_oneBQ_
-__RNvMs_NtNtCs8Z8q0vUHYys_12clap_builder6parser6parserNtB4_6Parser16get_matches_with
-__RNvMs_NtNtCs8Z8q0vUHYys_12clap_builder6parser6parserNtB4_6Parser15resolve_pending
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner15try_allocate_inCs8Z8q0vUHYys_12clap_builder
-__RNvMs_NtNtCs8Z8q0vUHYys_12clap_builder6parser6parserNtB4_6Parser5react
-__RNvMNtNtCs8Z8q0vUHYys_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher6remove
-__RNvMs_NtNtCs8Z8q0vUHYys_12clap_builder6parser6parserNtB4_6Parser16start_custom_arg
-__RNvXsc_NtNtCs8Z8q0vUHYys_12clap_builder7builder12value_parserNtB5_15BoolValueParserNtB5_14AnyValueParser7type_idB9_
-__RNvMs0_NtNtCs8Z8q0vUHYys_12clap_builder4util8flat_mapINtB5_5EntryNtNtB7_2id2IdNtNtNtNtB9_6parser7matches11matched_arg10MatchedArgE9or_insertB9_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11matched_arg10MatchedArgE8grow_oneBU_
-__RNvMNtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11matched_argNtB2_10MatchedArg13new_val_group
-__RNvXNtCs7yUGMWzZmiN_8clap_lex3extNtNtNtCs82bWklYMk3w_3std3ffi6os_str5OsStrNtB2_8OsStrExt12strip_prefix
-__RNvMs1_Cs7yUGMWzZmiN_8clap_lexNtB5_9ParsedArg8to_short
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecNtNtNtCs8Z8q0vUHYys_12clap_builder4util9any_value8AnyValueEE8grow_oneB18_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecNtNtNtCs82bWklYMk3w_3std3ffi6os_str8OsStringEE8grow_oneCs8Z8q0vUHYys_12clap_builder
-__RNvMs_NtNtCs8Z8q0vUHYys_12clap_builder6parser6parserNtB4_6Parser15push_arg_values
-__RNvXsC_NtNtCs8Z8q0vUHYys_12clap_builder7builder12value_parserNtB5_15BoolValueParserNtB5_16TypedValueParser9parse_ref
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs8Z8q0vUHYys_12clap_builder4util9any_value8AnyValueE8grow_oneBS_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecjE8grow_oneCs8Z8q0vUHYys_12clap_builder
-__RNvMs_NtNtCs8Z8q0vUHYys_12clap_builder6parser6parserNtB4_6Parser12add_defaults
-__RNvXsc_NtNtCs8Z8q0vUHYys_12clap_builder7builder12value_parserNtB5_15BoolValueParserNtB5_14AnyValueParser10parse_ref_B9_
-__RNvMNtNtCs8Z8q0vUHYys_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher10add_val_to
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs8Z8q0vUHYys_12clap_builder6parser6parser11ParseResultEBH_
-__RNvXsc_NtNtCs8Z8q0vUHYys_12clap_builder7builder12value_parserINtB5_20RangedI64ValueParserhENtB5_14AnyValueParser7type_idCsdp43gqXvMwI_11weavepy_cli
-__RNvXsu_NtNtCs8Z8q0vUHYys_12clap_builder7builder12value_parserINtB5_20RangedI64ValueParserhENtB5_16TypedValueParser9parse_refCsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs8Z8q0vUHYys_12clap_builder4util5graph10ChildGraphNtNtBG_2id2IdEEBI_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11matched_arg10MatchedArgEEB1g_
-__RNvMNtNtCs8Z8q0vUHYys_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher10into_inner
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtCs8Z8q0vUHYys_12clap_builder6parser6parser10PendingArgEEB13_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs8Z8q0vUHYys_12clap_builder7builder7command7CommandECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtNtCs8Z8q0vUHYys_12clap_builder7builder3str3StrbEEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs8Z8q0vUHYys_12clap_builder7builder3arg3ArgECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtCs8Z8q0vUHYys_12clap_builder7builder12value_parser11ValueParserEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtNtCs8Z8q0vUHYys_12clap_builder4util2id2IdNtNtNtB1f_7builder13arg_predicate12ArgPredicateINtNtB4_6option6OptionIBC_NtNtB1W_6os_str5OsStrEEEEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs8Z8q0vUHYys_12clap_builder7builder3ext10ExtensionsECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs8Z8q0vUHYys_12clap_builder4util9any_value8AnyValueEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs8Z8q0vUHYys_12clap_builder7mkeymap3KeyEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs8Z8q0vUHYys_12clap_builder7builder7command7CommandEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs8Z8q0vUHYys_12clap_builder7builder9arg_group8ArgGroupEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs8Z8q0vUHYys_12clap_builder7builder9arg_group8ArgGroupECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtCs8Z8q0vUHYys_12clap_builder7builder12value_parser11ValueParserEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs8Z8q0vUHYys_12clap_builder7builder3ext10ExtensionsECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs8Z8q0vUHYys_12clap_builder4util9any_value8AnyValueEECsdp43gqXvMwI_11weavepy_cli
-__RNvXs2_Csdp43gqXvMwI_11weavepy_cliNtB5_3CliNtNtCs8Z8q0vUHYys_12clap_builder6derive14FromArgMatches20from_arg_matches_mut
-__RINvMs0_NtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches14try_remove_onebECsdp43gqXvMwI_11weavepy_cli
-__RINvMNtNtCs8Z8q0vUHYys_12clap_builder4util8flat_mapINtB3_7FlatMapNtNtB5_2id2IdNtNtNtNtB7_6parser7matches11matched_arg10MatchedArgE12remove_entryeECsdp43gqXvMwI_11weavepy_cli
-__RNvMNtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11matched_argNtB2_10MatchedArg17into_vals_flatten
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecIBC_NtNtNtCs82bWklYMk3w_3std3ffi6os_str8OsStringEEECs8Z8q0vUHYys_12clap_builder
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs82bWklYMk3w_3std3ffi6os_str8OsStringEECs8Z8q0vUHYys_12clap_builder
-__RNvXsi_NtNtNtCs8Mbv00yxnRz_4core4iter8adapters7flattenINtB5_13FlattenCompatINtNtNtCshxvaOLs88l5_5alloc3vec9into_iter8IntoIterINtB1g_3VecNtNtNtCs8Z8q0vUHYys_12clap_builder4util9any_value8AnyValueEEIB1c_B2a_EENtNtNtB9_6traits8iterator8Iterator4nextCsdp43gqXvMwI_11weavepy_cli
-__RNvXNtCs8Mbv00yxnRz_4core3anybNtB2_3Any7type_idCs8Z8q0vUHYys_12clap_builder
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtNtB4_4iter8adapters7flatten7FlattenINtNtNtCshxvaOLs88l5_5alloc3vec9into_iter8IntoIterINtB1m_3VecNtNtNtCs8Z8q0vUHYys_12clap_builder4util9any_value8AnyValueEEEECsdp43gqXvMwI_11weavepy_cli
-__RINvMNtNtCs8Z8q0vUHYys_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs8Mbv00yxnRz_4core6option6OptionbEECsdp43gqXvMwI_11weavepy_cli
-__RINvMs0_NtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches14try_remove_onehECsdp43gqXvMwI_11weavepy_cli
-__RNvXNtCs8Mbv00yxnRz_4core3anyhNtB2_3Any7type_idCsdp43gqXvMwI_11weavepy_cli
-__RINvMNtNtCs8Z8q0vUHYys_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs8Mbv00yxnRz_4core6option6OptionhEECsdp43gqXvMwI_11weavepy_cli
-__RINvMs0_NtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches15try_remove_manyNtNtCshxvaOLs88l5_5alloc6string6StringECsdp43gqXvMwI_11weavepy_cli
-__RINvMNtNtCs8Z8q0vUHYys_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtNtB5_7matches11arg_matches6ValuesNtNtCshxvaOLs88l5_5alloc6string6StringEEECsdp43gqXvMwI_11weavepy_cli
-__RINvMs0_NtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches14try_remove_oneNtNtCshxvaOLs88l5_5alloc6string6StringECsdp43gqXvMwI_11weavepy_cli
-__RINvMNtNtCs8Z8q0vUHYys_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCshxvaOLs88l5_5alloc6string6StringEECsdp43gqXvMwI_11weavepy_cli
-__RINvMs0_NtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches14try_remove_oneNtNtCs82bWklYMk3w_3std4path7PathBufECsdp43gqXvMwI_11weavepy_cli
-__RINvMNtNtCs8Z8q0vUHYys_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCs82bWklYMk3w_3std4path7PathBufEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11arg_matches10ArgMatchesECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs8Z8q0vUHYys_12clap_builder4util2id2IdEECsdp43gqXvMwI_11weavepy_cli
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons13raw_arg_bytes
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins16bridge_to_object
-__RINvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB7_6Object8from_strReEB9_
-__RNvXsq_NtCs2D882usVKTN_10weavepy_vm12shared_valueNtB5_9SharedStrINtNtCs8Mbv00yxnRz_4core7convert4FromReE4from
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RINvNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECsdp43gqXvMwI_11weavepy_cli
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsdp43gqXvMwI_11weavepy_cli
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm12shared_value12SliceStoragehEE9drop_slowBL_
-__RNvCsdp43gqXvMwI_11weavepy_cli11build_flags
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECsdp43gqXvMwI_11weavepy_cli
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VechENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCsdp43gqXvMwI_11weavepy_cli
-__RNvXso_Cs2D882usVKTN_10weavepy_vmNtB5_16InterpreterFlagsNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvCsdp43gqXvMwI_11weavepy_cli23run_source_with_options
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockFEuE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zECs2D882usVKTN_10weavepy_vm
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockFEuE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0Cs2D882usVKTN_10weavepy_vm
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockFG_RL0_INtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtB15_6string6StringEE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEB1C_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockFG_RL0_INtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtB1S_6string6StringEE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B2p_
-__RNvMCsdp43gqXvMwI_11weavepy_cliNtB2_12EnvOverrides8from_env
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockFG_RL0_INtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEINtNtCs8Mbv00yxnRz_4core6option6OptionyEE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEB1C_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockFG_RL0_INtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEINtNtCs8Mbv00yxnRz_4core6option6OptionyEE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B2p_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockFG_RL0_eENtNtCs1LCFj3BjrAK_14weavepy_parser6parser21UnicodeNameResolutionE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEB16_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockFG_RL0_eENtNtCs1LCFj3BjrAK_14weavepy_parser6parser21UnicodeNameResolutionE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B1T_
-__RNvCs1LCFj3BjrAK_14weavepy_parser17parse_module_full
-__RNvNtCsftQRLRmRWkP_13weavepy_lexer7scanner21tokenize_with_escapes
-__RNvNtCsftQRLRmRWkP_13weavepy_lexer7scanner16tokenize_partial
-__RNvMNtCsftQRLRmRWkP_13weavepy_lexer7scannerNtB2_7Scanner29scan_ident_or_prefixed_string
-__RNvMs0_NtCsftQRLRmRWkP_13weavepy_lexer5tokenNtB5_7Keyword10from_ident
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCsftQRLRmRWkP_13weavepy_lexer5token5TokenE8grow_oneBQ_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsftQRLRmRWkP_13weavepy_lexer
-__RNvXsc_NtNtCs8Z8q0vUHYys_12clap_builder7builder12value_parserINtB5_20RangedI64ValueParserhENtB5_14AnyValueParser10parse_ref_Csdp43gqXvMwI_11weavepy_cli
-__RNvMs5_NtNtCs8Z8q0vUHYys_12clap_builder7builder7commandNtB5_7Command14required_graph
-__RNvMNtNtCs8Z8q0vUHYys_12clap_builder6parser9validatorNtB2_9Validator8validate
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs8Z8q0vUHYys_12clap_builder4util8flat_map7FlatMapNtNtBG_2id2IdINtNtCshxvaOLs88l5_5alloc3vec3VecB1w_EEEBI_
-__RNvMNtNtCs8Z8q0vUHYys_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher21fill_in_global_values
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs8Z8q0vUHYys_12clap_builder4util8flat_map7FlatMapNtNtBG_2id2IdNtNtNtNtBI_6parser7matches11matched_arg10MatchedArgEEBI_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECsdp43gqXvMwI_11weavepy_cli
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtB7_6string6StringENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs82bWklYMk3w_3std4path7PathBufEECsdp43gqXvMwI_11weavepy_cli
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once9call_onceNCNvNtCsl2apZsWCN6y_12weavepy_capi6loader27install_vm_extension_loader0E0B14_
-__RNvNtCs2D882usVKTN_10weavepy_vm10ext_loader24install_extension_loader
-__RINvNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs2D882usVKTN_10weavepy_vm
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecmE8grow_oneCsftQRLRmRWkP_13weavepy_lexer
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13codecs_engine11utf8_encode
-__RNvMs_NtNtCs2D882usVKTN_10weavepy_vm6stdlib13codecs_engineNtB4_6EncCtx3new
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib13codecs_engine6EncCtxEBH_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10codecs_mod12encoding_key
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RNvMs_NtNtCs8Z8q0vUHYys_12clap_builder6parser6parserNtB4_6Parser19possible_subcommand
-__RNvCs2D882usVKTN_10weavepy_vm17bridged_arg_bytes
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10codecs_mod10encode_obj
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10codecs_mod10encode_str
-__RNvCsdp43gqXvMwI_11weavepy_cli8run_path
-__RNvCs2D882usVKTN_10weavepy_vm25decode_source_bytes_inner
-__RNvCs2D882usVKTN_10weavepy_vm25translate_source_newlines
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs82bWklYMk3w_3std4path7PathBufNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECsdp43gqXvMwI_11weavepy_cli
-__RNvMNtCsftQRLRmRWkP_13weavepy_lexer7scannerNtB2_7Scanner11scan_string
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecThjEE8grow_oneCsftQRLRmRWkP_13weavepy_lexer
-__RNvMNtCsftQRLRmRWkP_13weavepy_lexer7scannerNtB2_7Scanner20verify_end_of_number
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser25parse_with_flufl_tracking
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser9new_inner
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser12parse_module
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser24skip_trivia_and_newlines
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser15parse_statement
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser24check_stmt_as_ifexp_body
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser16consume_stmt_end
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4StmtE8grow_oneBQ_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs1LCFj3BjrAK_14weavepy_parser
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser12apply_poison
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser4meta
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser6parser6ParserEBF_
-__RNvCs6MP6LIxl4z2_16weavepy_compiler27compile_module_with_options
-__RNvNvCs6MP6LIxl4z2_16weavepy_compiler22collect_module_imports4walk
-__RNvNvCs6MP6LIxl4z2_16weavepy_compiler21module_mentions_super7in_stmt
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4StmtNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs6MP6LIxl4z2_16weavepy_compiler
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler7ast_opt9fold_stmt
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler19compile_module_body
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler4emit
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs6MP6LIxl4z2_16weavepy_compiler
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler15resolve_colspan
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler12compile_stmt
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtB7_6string6StringENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvMNtCs6MP6LIxl4z2_16weavepy_compiler6internNtB3_11InternIndex6internNtB5_8ConstantNvB3_13constant_hashEB5_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantE8grow_oneBO_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecjE8grow_oneCs6MP6LIxl4z2_16weavepy_compiler
-__RNvCs1njKG4L9aB3_7___rustc19___rust_alloc_zeroed
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs6MP6LIxl4z2_16weavepy_compiler9flowgraph5BlockE8grow_oneBQ_
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB4_5Block10next_instr
-__RINvNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs0_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg13clear_visited
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code14map_to_cpython
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs6MP6LIxl4z2_16weavepy_compiler6intern11InternIndexEBF_
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler8validate15validate_module
-__RNvMNtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB2_5Scope3new
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs6MP6LIxl4z2_16weavepy_compiler8validate9ValidatorEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs6MP6LIxl4z2_16weavepy_compiler
-__RNvCs6MP6LIxl4z2_16weavepy_compiler25collect_global_decls_deep
-__RNvCs6MP6LIxl4z2_16weavepy_compiler19for_each_scope_expr
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs6MP6LIxl4z2_16weavepy_compiler8bytecode11InstructionE8grow_oneBQ_
-__RNvXsP_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8ConstantNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg20optimize_basic_block
-__RNvXs3_NtNtCs8Mbv00yxnRz_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg31remove_redundant_nops_and_jumps
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg5shape
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtBU_3Cfg18optimize_load_fast3RefE8grow_oneBW_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs6MP6LIxl4z2_16weavepy_compiler6intern11InternIndexEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCsblfCQaHOzq5_8indexmap3map8IndexMapNtNtCshxvaOLs88l5_5alloc6string6StringNtCs6MP6LIxl4z2_16weavepy_compiler7BindingEEB1U_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler9LoopFrameEEB1a_
-__RNvMs8_NtCs6MP6LIxl4z2_16weavepy_compiler8bytecodeNtB5_10CacheTable6resize
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtCshxvaOLs88l5_5alloc6string6StringuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcNtCs6MP6LIxl4z2_16weavepy_compiler9LineIndexE9drop_slowBF_
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcINtNtNtNtCs82bWklYMk3w_3std11collections4hash3set7HashSetNtNtB7_6string6StringEE9drop_slowCs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RceE9drop_slowCs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4StmtEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast8StmtKindECs6MP6LIxl4z2_16weavepy_compiler
-__RNvCs2D882usVKTN_10weavepy_vm18argv_str_to_object
-__RNvXsy_NtCs2D882usVKTN_10weavepy_vm12shared_valueNtB5_9SharedStrNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser22parse_subscript_single
-__RNvCs6MP6LIxl4z2_16weavepy_compiler43should_apply_two_element_slice_optimization
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler17starunpack_helper
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler19compile_load_target
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser11parse_while
-__RNvMs8_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_10CodeObject10content_eq
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast5AliasEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast3ArgEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast9ArgumentsECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast3ArgECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs6MP6LIxl4z2_16weavepy_compiler8validate9DirectiveEEB1c_
-__RNvMsd_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_9LineIndex3new
-__RNvCs6MP6LIxl4z2_16weavepy_compiler16collect_assigned
-__RNvCs6MP6LIxl4z2_16weavepy_compiler19collect_walrus_stmt
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler19register_comp_cells
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtCs6MP6LIxl4z2_16weavepy_compiler7ColSpanE8grow_oneBO_
-__RNvCs6MP6LIxl4z2_16weavepy_compiler21block_has_annotations
-__RNvCs6MP6LIxl4z2_16weavepy_compiler30block_has_deferred_annotations
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler18compile_stmt_inner
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler6finish
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler9flowgraph8optimize
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code13stack_effects
-__RNvMs0_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg5chain
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg18remove_unreachable
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg20resolve_line_numbers
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEBD_
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg27block_remove_redundant_nops
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEEB1a_
-__RNvCs6MP6LIxl4z2_16weavepy_compiler11make_params
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB4_9Validator10visit_stmt
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs6MP6LIxl4z2_16weavepy_compiler8validate5ScopeEBF_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner15try_allocate_inCs6MP6LIxl4z2_16weavepy_compiler
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler3new
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler12FinallyFrameEEB1a_
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg18optimize_load_fast
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code14cp_stack_shape
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs6MP6LIxl4z2_16weavepy_compiler9flowgraph5BlockEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionTNtNtCshxvaOLs88l5_5alloc6string6StringBY_BY_EEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler18DeferredAnnotationEEB1a_
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser19find_invalid_target
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser16parse_assign_rhs
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser12parse_number
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser22expect_or_forgot_comma
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser26take_trailing_type_comment
-__RNvMsr_NtCs8Mbv00yxnRz_4core3numx16from_ascii_radix
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB4_9Validator12visit_target
-__RNvCs6MP6LIxl4z2_16weavepy_compiler20collect_target_names
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftNtNtCshxvaOLs88l5_5alloc6string6StringNvYB1m_NtNtBa_3cmp10PartialOrd2ltECs6MP6LIxl4z2_16weavepy_compiler
-__RINvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB6_7HashMapNtNtCshxvaOLs88l5_5alloc6string6StringuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE12contains_keyBO_ECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast5CmpOpE8grow_oneBQ_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs1LCFj3BjrAK_14weavepy_parser
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRNtNtCshxvaOLs88l5_5alloc6string6StringECs6MP6LIxl4z2_16weavepy_compiler
-__RNvNvCs6MP6LIxl4z2_16weavepy_compiler21module_mentions_super7in_expr
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB4_9Validator22push_type_params_scope
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB4_9Validator14visit_function
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTReNtNtCsftQRLRmRWkP_13weavepy_lexer5token4SpanEE8grow_oneCs6MP6LIxl4z2_16weavepy_compiler
-__RNvCs6MP6LIxl4z2_16weavepy_compiler19collect_walrus_expr
-__RINvXs8_NtCs3fRkGY2FHAU_9hashbrown3setINtB6_7HashSetNtNtCshxvaOLs88l5_5alloc6string6StringNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateEINtNtNtNtCs8Mbv00yxnRz_4core4iter6traits7collect6ExtendBO_E6extendINtNtNtNtB1w_11collections4hash3set7HashSetBO_EECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler25compile_default_arguments
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler18emit_make_function
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast8ExprKindECs6MP6LIxl4z2_16weavepy_compiler
-__RNvCs6MP6LIxl4z2_16weavepy_compiler27collect_reads_assign_target
-__RNvXs1b_NtCs1LCFj3BjrAK_14weavepy_parser3astNtB6_8ConstantNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvXsa_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8ConstantINtNtCs8Mbv00yxnRz_4core7convert4FromNtNtCs1LCFj3BjrAK_14weavepy_parser3ast8ConstantE4from
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler14unwind_fblocks
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler7emit_at
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler18close_unwind_holes
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler22super_attr_optimizable
-__RINvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_expr_inners4_0EB6_
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg20const_loading_instrs
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast8ConstantECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler35import_flag_shadowed_by_module_comp
-__RINvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_expr_inners2_0EB6_
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler15compile_jump_if
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEBD_
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler24genexp_call_optimization
-__RNCNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB7_8Compiler18compile_expr_inners1_0B7_
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler15compile_compare
-__RNvCs6MP6LIxl4z2_16weavepy_compiler11emit_cmp_op
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler10patch_jump
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRmECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg36is_exit_or_eval_check_without_lineno
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser26parse_target_list_no_tuple
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser20parse_target_or_star
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser9parse_for
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapmuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCs6MP6LIxl4z2_16weavepy_compiler
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmuEE14reserve_rehashNCINvNtB8_3map11make_hashermuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtCs6MP6LIxl4z2_16weavepy_compiler9LoopFrameE8grow_oneBO_
-__RNvMs1_NtCsftQRLRmRWkP_13weavepy_lexer5tokenNtB5_12StringPrefix5parse
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser22parse_simple_statement
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser21parse_expression_list
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser24parse_ternary_or_starred
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser13parse_ternary
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser8parse_or
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser9parse_and
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser9parse_not
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser16parse_comparison
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser12parse_bit_or
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser13parse_bit_xor
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser13parse_bit_and
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser11parse_shift
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser12parse_addsub
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser12parse_muldiv
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser11parse_unary
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser11parse_power
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser10parse_atom
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser13string_prefix
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser6lexeme
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser13decode_string
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser12strip_quotes
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser15decode_str_body
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser15cps_to_constant
-__RINvNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs1LCFj3BjrAK_14weavepy_parser
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser16constant_str_cps
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser24check_string_expr_string
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser19at_expression_start
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser5check
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEEB1c_
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser12parse_import
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser17parse_dotted_name
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser6expect
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser5ident
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser3eat
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser24check_import_alias_debug
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast5AliasE8grow_oneBQ_
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser18parse_function_def
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser26collect_pep695_type_params
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser21parse_arguments_inner
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser18try_arg_annotation
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast3ArgE8grow_oneBQ_
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser11parse_block
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecReE8grow_oneCs1LCFj3BjrAK_14weavepy_parser
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprE8grow_oneBQ_
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser23take_colon_type_comment
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser27take_def_body_type_comments
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser12parse_return
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser8parse_if
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser15parse_call_args
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapNtNtCshxvaOLs88l5_5alloc6string6StringuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB4_9Validator10visit_expr
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB4_9Validator13mark_assigned
-__RNvCs6MP6LIxl4z2_16weavepy_compiler13collect_decls
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs6MP6LIxl4z2_16weavepy_compiler8validate5ScopeE8grow_oneBQ_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCshxvaOLs88l5_5alloc6string6StringECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB4_9Validator10visit_body
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs6MP6LIxl4z2_16weavepy_compiler8validate5ScopeEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs6MP6LIxl4z2_16weavepy_compiler8validate9DirectiveEEB1c_
-__RNvXsZ_NtCs1LCFj3BjrAK_14weavepy_parser3astNtB5_8ExprKindNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvXs1b_NtCs1LCFj3BjrAK_14weavepy_parser3astNtB6_8ConstantNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast5AliasENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCs6MP6LIxl4z2_16weavepy_compiler
-__RNvXsM_NtCs1LCFj3BjrAK_14weavepy_parser3astNtB5_9ArgumentsNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs1LCFj3BjrAK_14weavepy_parser3ast3ArgNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs6MP6LIxl4z2_16weavepy_compiler
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs6MP6LIxl4z2_16weavepy_compiler
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast9TypeParamENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCs6MP6LIxl4z2_16weavepy_compiler
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs1LCFj3BjrAK_14weavepy_parser3ast7KeywordNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs6MP6LIxl4z2_16weavepy_compiler
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler7ast_opt9fold_expr
-__RNvMs2_NtCsblfCQaHOzq5_8indexmap3mapINtB5_8IndexMapNtNtCshxvaOLs88l5_5alloc6string6StringNtCs6MP6LIxl4z2_16weavepy_compiler7BindingE11insert_fullB1r_
-__RNvMs_NtCsblfCQaHOzq5_8indexmap5innerINtB4_4CoreNtNtCshxvaOLs88l5_5alloc6string6StringNtCs6MP6LIxl4z2_16weavepy_compiler7BindingE10push_entryB1o_
-__RNCNvCs6MP6LIxl4z2_16weavepy_compiler32collect_comp_walrus_targets_stmt0B3_
-__RNvNvCs6MP6LIxl4z2_16weavepy_compiler32collect_comp_walrus_targets_stmt4walk
-__RNCNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB7_8Compiler23collect_comp_cells_stmt0B7_
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler23collect_comp_cells_expr
-__RNvCs6MP6LIxl4z2_16weavepy_compiler15clean_docstring
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecReEINtB2_18SpecFromIterNestedB11_INtNtNtCs8Mbv00yxnRz_4core3str4iter5SplitcEE9from_iterCs6MP6LIxl4z2_16weavepy_compiler
-__RNvXs9_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8ConstantNtNtCs8Mbv00yxnRz_4core3cmp9PartialEq2eq
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler15emit_store_name
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler18classify_for_store
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCshxvaOLs88l5_5alloc6string6StringNtCs6MP6LIxl4z2_16weavepy_compiler7BindingE3geteEB1s_
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneReECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler26compile_function_def_inner
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler26build_function_object_full
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler19child_qualname_impl
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtB6_6string6StringEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs82bWklYMk3w_3std11collections4hash3set8IntoIterB11_EE9from_iterCs6MP6LIxl4z2_16weavepy_compiler
-__RNvCs6MP6LIxl4z2_16weavepy_compiler19stmt_contains_yield
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler9free_scan
-__RNvCs6MP6LIxl4z2_16weavepy_compiler19expr_contains_yield
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler22analyze_scope_function
-__RNvXNtCs3fRkGY2FHAU_9hashbrown3mapINtB2_7HashMapNtNtCshxvaOLs88l5_5alloc6string6StringuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCs6MP6LIxl4z2_16weavepy_compiler
-__RNvCs6MP6LIxl4z2_16weavepy_compiler21collect_reads_stmt_in
-__RNvCs6MP6LIxl4z2_16weavepy_compiler26collect_reads_stmt_in_impl
-__RNvCs6MP6LIxl4z2_16weavepy_compiler18collect_reads_expr
-__RNvCs6MP6LIxl4z2_16weavepy_compiler18collect_inner_free
-__RNvCs6MP6LIxl4z2_16weavepy_compiler23collect_inner_free_expr
-__RNvCs6MP6LIxl4z2_16weavepy_compiler27collect_pep695_header_reads
-__RNvCs6MP6LIxl4z2_16weavepy_compiler23collect_inner_free_impl
-__RINvMsj_NtCshxvaOLs88l5_5alloc3vecINtB6_3VecNtNtB8_6string6StringE16extend_desugaredINtNtNtNtCs82bWklYMk3w_3std11collections4hash3set8IntoIterBG_EECs6MP6LIxl4z2_16weavepy_compiler
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCshxvaOLs88l5_5alloc6string6StringNtCs6MP6LIxl4z2_16weavepy_compiler7BindingE12get_index_ofBO_EB1s_
-__RNCNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB7_8Compiler22analyze_scope_functions0_0B7_
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler28collect_uses_through_inlined
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler18compile_expr_inner
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler14emit_load_name
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler16var_index_or_add
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler14compile_assign
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg21scan_block_for_locals
-__RNvNtNtCs8Z8q0vUHYys_12clap_builder6parser9validator23gather_direct_conflicts
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecNtNtNtCs8Z8q0vUHYys_12clap_builder4util2id2IdEE8grow_oneB18_
-__RNvMs_NtNtCs8Z8q0vUHYys_12clap_builder6parser9validatorNtB4_9Conflicts16gather_conflicts
-__RNvXsf_NtNtCs8Mbv00yxnRz_4core5slice3cmpNtNtNtCs8Z8q0vUHYys_12clap_builder4util2id2IdNtB5_13SliceContains14slice_containsBI_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecRNtNtNtCs8Z8q0vUHYys_12clap_builder4util2id2IdE8grow_oneBT_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtCs8Z8q0vUHYys_12clap_builder6parser7matches11matched_arg10MatchedArgECsdp43gqXvMwI_11weavepy_cli
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcDNtNtCs8Mbv00yxnRz_4core3any3AnyNtNtBL_6marker4SyncNtB1e_4SendEL_E9drop_slowCs8Z8q0vUHYys_12clap_builder
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc4sync4WeakDNtNtB4_3any3AnyNtNtB4_6marker4SyncNtB1s_4SendEL_RNtNtBG_5alloc6GlobalEECs8Z8q0vUHYys_12clap_builder
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecIBC_NtNtNtCs82bWklYMk3w_3std3ffi6os_str8OsStringEEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs82bWklYMk3w_3std3ffi6os_str8OsStringEECsdp43gqXvMwI_11weavepy_cli
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser19simple_keyword_stmt
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser14parse_comp_for
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler8validate20collect_name_targets
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB4_9Validator13record_walrus
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast13ComprehensionE8grow_oneBQ_
-__RNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB4_9Validator19visit_comprehension
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler8validate22collect_walrus_targets
-__RNvNvCs6MP6LIxl4z2_16weavepy_compiler27collect_comp_scope_walruses5visit
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs1LCFj3BjrAK_14weavepy_parser3ast13ComprehensionNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler17compile_class_def
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler19compile_class_value
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler16build_class_body
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEBD_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtCs6MP6LIxl4z2_16weavepy_compiler10ExcHandlerE8grow_oneBO_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecINtNtB7_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEE8grow_oneB1n_
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types23install_descriptor_init
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject13set_metaclass
-__RNvCs6MP6LIxl4z2_16weavepy_compiler25body_reads_super_or_class
-__RNCNvNCNvCs6MP6LIxl4z2_16weavepy_compiler25body_reads_super_or_class09any_class0B7_
-__RNvCs6MP6LIxl4z2_16weavepy_compiler19function_free_names
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler18cell_or_free_index
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4StmtEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast9TypeParamEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast7KeywordEECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser15parse_class_def
-__RNCNvMs_NtCs6MP6LIxl4z2_16weavepy_compiler8validateNtB6_9Validator10visit_stmt0B8_
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4StmtNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs6MP6LIxl4z2_16weavepy_compiler
-__RNvXsM_NtCs1LCFj3BjrAK_14weavepy_parser3astNtB5_9ArgumentsNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler6mangle17mangle_class_body
-__RNvXs1i_NtCs8Mbv00yxnRz_4core3fmtReNtB6_7Display3fmtCs6MP6LIxl4z2_16weavepy_compiler
-__RNvMNtCs6MP6LIxl4z2_16weavepy_compiler6mangleNtB2_7Mangler4stmt
-__RNvMNtCs6MP6LIxl4z2_16weavepy_compiler6mangleNtB2_7Mangler4name
-__RNvMNtCs6MP6LIxl4z2_16weavepy_compiler6mangleNtB2_7Mangler9arguments
-__RNvMNtCs6MP6LIxl4z2_16weavepy_compiler6mangleNtB2_7Mangler4expr
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs6MP6LIxl4z2_16weavepy_compiler6mangle7ManglerEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCshxvaOLs88l5_5alloc6string6StringECs6MP6LIxl4z2_16weavepy_compiler
-__RNCNvCs6MP6LIxl4z2_16weavepy_compiler32class_body_defs_claim_class_cell0B3_
-__RNvCs6MP6LIxl4z2_16weavepy_compiler23method_references_class
-__RNvCs6MP6LIxl4z2_16weavepy_compiler20stmt_needs_classdict
-__RNvCs6MP6LIxl4z2_16weavepy_compiler23class_body_reads_dunder
-__RNvNvCs6MP6LIxl4z2_16weavepy_compiler23class_body_reads_dunder10stmt_reads
-__RNvXs1_Cs2D882usVKTN_10weavepy_vmNtB5_11InterpreterNtNtCs8Mbv00yxnRz_4core7default7Default7default
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10locale_mod13init_from_env
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins16default_builtins
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object11from_static
-__RNvMs2_NtCsblfCQaHOzq5_8indexmap3mapINtB5_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBP_6ObjectNtNtBR_8fasthash13FxBuildHasherE11insert_fullBR_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object19py_hash_bytes_slice
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object8HashAlgoE10initializeNCINvB2_11get_or_initNCNvBV_9hash_algo0E0zEBX_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object8HashAlgoE10initializeNCINvB1a_11get_or_initNCNvB1I_9hash_algo0E0zE0E0B1K_
-__RNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner17try_reserve_exactCs2D882usVKTN_10weavepy_vm
-__RNvXsz_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_7DictKeyNtNtCs8Mbv00yxnRz_4core3cmp9PartialEq2eq
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object7is_same
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object8eq_value
-__RNvNtCs2D882usVKTN_10weavepy_vm6object19key_needs_interp_eq
-__RNvNtCs2D882usVKTN_10weavepy_vm6object26instance_has_custom_dunder
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCNvMNtBa_9lazy_lockINtB1a_8LazyLockINtNtCs1jOiKDdKJOr_8lock_api6rwlock6RwLockNtNtCseaAvxSPddma_11parking_lot10raw_rwlock9RawRwLockINtNtNtNtBc_11collections4hash3set7HashSetjEEE5force0E0Cs2D882usVKTN_10weavepy_vm
-__RINvYNtNtCs2D882usVKTN_10weavepy_vm8fasthash13FxBuildHasherNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRNtNtB7_6object7DictKeyEB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object11py_str_hash
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object9BuiltinFnE9drop_slowBK_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object9BuiltinFnEBF_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object13py_hash_value
-__RINvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB6_7HashMapjuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6removejECs2D882usVKTN_10weavepy_vm
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRjECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCshxvaOLs88l5_5alloc5boxed3BoxDG0_INtNtNtB4_3ops8function2FnTRL1_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectRL0_STNtNtB12_6string6StringB26_EEEp6OutputINtNtB4_6result6ResultB26_NtNtB2a_5error12RuntimeErrorENtNtB4_6marker4SyncNtB4q_4SendEL_EEEB2a_
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons19interactive_printer
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvNtB1K_13vm_singletons15not_implemented0E0zE0E0B1K_
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE12get_index_ofBO_EBS_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEE8grow_oneB18_
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry16mark_default_new
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry8note_key
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry23register_text_signature
-__RNvXsq_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RNvXs1i_NtCs8Mbv00yxnRz_4core3fmtReNtB6_7Display3fmtCs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjNtNtCs2D882usVKTN_10weavepy_vm14descr_registry10StoredMetaEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBV_
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertBS_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBV_
-__RNvMsg_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_13MethodWrapper3new
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types20install_type_dunders
-__RNvXNtNtNtCs8Mbv00yxnRz_4core5array4iter10iter_innerSINtNtNtB8_3mem12maybe_uninit11MaybeUninitTReNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEENtB2_11PartialDrop12partial_dropB1B_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins20function_get_builtin
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types24install_gen_name_getsets
-__RINvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB7_6Object8from_strNtNtCshxvaOLs88l5_5alloc6string6StringEB9_
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject13new_exception
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types8exc_slot
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtB1j_13tuple_storage12TupleStorageEuE16get_or_init_slowNvNvNvMs1j_NtB1j_6objectNtB3c_6Object9new_tuple11EMPTY_TUPLE27___rust_std_internal_init_fnEB1j_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RNvNvMNtCs2D882usVKTN_10weavepy_vm13builtin_typesNtB4_12BuiltinTypes5build22install_field_defaults
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types28install_syntax_error_dunders3set
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types29install_unicode_error_dunders
-__RNvMNtCs2D882usVKTN_10weavepy_vm13builtin_typesNtB2_12BuiltinTypes10as_globals
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterTNtNtB9_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB1m_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types29install_numeric_class_methods7install
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types29install_numeric_class_methods14install_method
-__RNvNtCs2D882usVKTN_10weavepy_vm12type_surface7install
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19builtin_type_dunder
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapNtNtCshxvaOLs88l5_5alloc6string6StringINtNtB1j_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEuE16get_or_init_slowNvNvNvNtB3y_8builtins19builtin_type_dunder12DUNDER_CACHE27___rust_std_internal_init_fnEB3y_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14unbound_method
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_9slot_callEB4_
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRNtNtCshxvaOLs88l5_5alloc6string6StringECs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtCshxvaOLs88l5_5alloc6string6StringINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1s_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB28_
-__RNvNtCs2D882usVKTN_10weavepy_vm12type_surface16insert_if_absent
-__RNvNtCs2D882usVKTN_10weavepy_vm12type_surface7builtin
-__RNvNtCs2D882usVKTN_10weavepy_vm12type_surface21install_named_methods
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins13lookup_method
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_9str_splitEB4_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCshxvaOLs88l5_5alloc5boxed3BoxDG0_INtNtNtB4_3ops8function2FnTRL1_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectRL0_STNtNtB12_6string6StringB26_EEEp6OutputINtNtB4_6result6ResultB26_NtNtB2a_5error12RuntimeErrorENtNtB4_6marker4SyncNtB4q_4SendEL_EEEB2a_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_10str_rsplitEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_14str_splitlinesEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_14str_replace_kwEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_17str_expandtabs_kwEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_10str_encodeEB4_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNCNvNtCs2D882usVKTN_10weavepy_vm12type_surface11unwrap_shim0EBH_
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object8new_list
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtBG_13tuple_storage12TupleStorageEEBG_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcINtBW_12SliceStoragehEEE10initializeNCINvB2_11get_or_initNCINvMs1j_NtBY_6objectNtB2P_6Object9new_bytesINtNtCshxvaOLs88l5_5alloc3vec3VechEE0E0zEBY_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_15bytes_rsplit_kwEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_19bytes_splitlines_kwEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_16bytes_expandtabsEB4_
-__RINvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB7_6Object13new_bytearrayINtNtCshxvaOLs88l5_5alloc3vec3VechEEB9_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface23install_numeric_getsets6getset
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object9PyComplexE9drop_slowBK_
-__RNvNtCs2D882usVKTN_10weavepy_vm12type_surface25register_descriptor_kinds
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RNvMsg_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_13MethodWrapper4func
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry6lookup
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry20is_slot_wrapper_name
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm14descr_registry9DescrMetaEBF_
-__RNvNtCs2D882usVKTN_10weavepy_vm12type_surface26install_docs_table_surface
-__RNvNtCs2D882usVKTN_10weavepy_vm17builtin_docs_data15builtin_doc_for
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtCshxvaOLs88l5_5alloc6string6StringINtNtBU_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1s_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1N_
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry6forget
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxDG_INtNtNtB4_3ops8function2FnTRL0_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEp6OutputINtNtB4_6result6ResultB1J_NtNtB1N_5error12RuntimeErrorENtNtB4_6marker4SyncNtB3w_4SendEL_EEB1N_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons15not_implemented
-__RNvMNtCs2D882usVKTN_10weavepy_vm13builtin_typesNtB2_12BuiltinTypes5build
-__RNvNtCs2D882usVKTN_10weavepy_vm5types10compute_c3
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEEB1t_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEEB1t_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEE8grow_oneB17_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCNvMNtBa_9lazy_lockINtB1a_8LazyLockINtNtCs1jOiKDdKJOr_8lock_api6rwlock6RwLockNtNtCseaAvxSPddma_11parking_lot10raw_rwlock9RawRwLockINtNtNtNtBc_11collections4hash3map7HashMapjReEEE5force0E0Cs2D882usVKTN_10weavepy_vm
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjReNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjReEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry8register
-__RNvYNCNvNtCs2D882usVKTN_10weavepy_vm14descr_registry10DESCR_META0INtNtNtCs8Mbv00yxnRz_4core3ops8function6FnOnceuE9call_onceB8_
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry18set_builtin_module
-__RNvMsg_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_13MethodWrapper16wrapper_dict_for
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types20install_type_dunders9mk_getset
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRjECs6MP6LIxl4z2_16weavepy_compiler
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types22install_module_methods7builtin
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object9new_tuple
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtBZ_13tuple_storage12TupleStorageEE4withNvYBU_NtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBU_EBZ_
-__RINvMNtCs2D882usVKTN_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj0_EB5_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtB7_3vec3VecNtNtBL_6object6ObjectEEE9drop_slowBL_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object8new_dict
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtBL_6object8DictDataEE9drop_slowBL_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object8DictDataEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtCsblfCQaHOzq5_8indexmap6BucketNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB1H_6ObjectEEEB1J_
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object7new_set
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtCsblfCQaHOzq5_8indexmap3set8IndexSetNtNtBL_6object7DictKeyNtNtBL_8fasthash13FxBuildHasherEEE9drop_slowBL_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtCsblfCQaHOzq5_8indexmap6BucketNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyuEEEB1J_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtCsblfCQaHOzq5_8indexmap6BucketNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyuEEEB1J_
-__RINvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB7_6Object9new_bytesINtNtCshxvaOLs88l5_5alloc3vec3VechEEB9_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcINtB1J_12SliceStoragehEEE10initializeNCINvB1a_11get_or_initNCINvMs1j_NtB1L_6objectNtB3E_6Object9new_bytesINtNtCshxvaOLs88l5_5alloc3vec3VechEE0E0zE0E0B1L_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_15bytes_decode_kwEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_12bytes_hex_kwEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_14bytes_split_kwEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_16bytes_replace_kwEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_18bytes_translate_kwEB4_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtB7_3vec3VechEEE9drop_slowBL_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14numeric_dunder
-__RNvXs1_NtNtNtCs8Mbv00yxnRz_4core3ops8function5implsQNCNvNtCs2D882usVKTN_10weavepy_vm12type_surface25register_descriptor_kinds0INtB7_5FnMutTTRNtNtBU_6object7DictKeyRNtB2h_6ObjectEEE8call_mutBU_
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types30install_builtin_descriptor_get
-__RNvCs2D882usVKTN_10weavepy_vm16builtin_type_doc
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry17mark_surface_only
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types13builtin_types
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject14new_with_flags
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE3getBO_EBS_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecIBC_INtNtBG_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEEEB1x_
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types16make_default_new
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjuEE14reserve_rehashNCINvNtB8_3map11make_hasherjuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface21install_value_richcmp15richcmp_builtin
-__RNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface29install_introspection_getsets6getset
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object11new_complex
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvXs1i_NtCs8Mbv00yxnRz_4core3fmtRNtNtCs2D882usVKTN_10weavepy_vm12shared_value9SharedStrNtB6_7Display3fmtBA_
-__RNvXsf_NtNtCs8Mbv00yxnRz_4core5slice3cmpReNtB5_13SliceContains14slice_containsCs2D882usVKTN_10weavepy_vm
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneReECs2D882usVKTN_10weavepy_vm
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE12get_index_ofNtBQ_6StrKeyEBS_
-__RNvCs2D882usVKTN_10weavepy_vm20builtin_slot_wrapper
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE3getNtBQ_6StrKeyEBS_
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter8run_site
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14run_site_inner
-__RINvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB7_6Object18new_frozenset_fromINtNtCshxvaOLs88l5_5alloc3vec3VecBG_EEB9_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object12FrozenSetObjE9drop_slowBK_
-__RNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB5_8JitState12get_compiled
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRPNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTTjNtNtCshxvaOLs88l5_5alloc6string6StringENtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1v_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1z_
-__RNvMs_NtCsblfCQaHOzq5_8indexmap5innerINtB4_4CoreNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBM_6ObjectE7reserveBO_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object13MethodWrapperE9drop_slowBK_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtBL_6object10PyIteratorEE9drop_slowBL_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object10PyIteratorEBF_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm13tuple_storage12TupleStorageE9drop_slowBK_
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapTjNtNtCshxvaOLs88l5_5alloc6string6StringENtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEuE16get_or_init_slowNvNvB3e_18SLOT_WRAPPER_CACHE27___rust_std_internal_init_fnEB3e_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterINtNtB9_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB1h_
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRTjNtNtCshxvaOLs88l5_5alloc6string6StringEECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins8class_of
-__RNvXs_NtNtNtCs8Mbv00yxnRz_4core5array4iter10iter_innerAINtNtNtBa_3mem12maybe_uninit11MaybeUninitTNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB1z_6ObjectEEj1_NtB4_11PartialDrop12partial_dropB1B_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtCsblfCQaHOzq5_8indexmap6BucketNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB1a_6ObjectEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm13tuple_storage12TupleStorageRNtNtBG_5alloc6GlobalEEB1e_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_12int_to_bytesEB4_
-__RNvMsh_NtCs2D882usVKTN_10weavepy_vm12shared_valueINtB5_7ThinArcINtB5_12SliceStoragehEE14from_vec_ownedB7_
-__RNvMsd_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_12PyMemoryView10from_bytes
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object12PyMemoryViewE9drop_slowBK_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTPNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectNtNtCs2D882usVKTN_10weavepy_vm5tier210CacheEntryEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1B_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1F_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier218materialize_parked
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14run_activation
-__RINvNtCs2D882usVKTN_10weavepy_vm8gc_trace10with_statebNvMs2_B2_NtB2_7GcState19has_any_finalizableEB4_
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize18init_stats_enabled
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter9quiet_run
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter10leaf_burst
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcDNtNtCs8Mbv00yxnRz_4core3any3AnyNtNtB2j_6marker4SyncNtB2M_4SendEL_EE10initializeNCINvB1a_11get_or_initNCNvCs2D882usVKTN_10weavepy_vm11code_vm_ext0E0zE0E0B3W_
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtB2p_5slice4iter4IterNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantENvB17_27constant_to_object_internedEE9from_iterB17_
-__RNvXsP_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8ConstantNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvCs2D882usVKTN_10weavepy_vm18constant_to_object
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter4step
-__RNvMCs2D882usVKTN_10weavepy_vmNtB2_5Frame3pop
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter7name_at
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21globals_missing_owner
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6result6ResultNtCs2D882usVKTN_10weavepy_vm11StepOutcomeNtNtBZ_5error12RuntimeErrorEEBZ_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os10build_path
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter9do_import
-__RNvNtCs2D882usVKTN_10weavepy_vm6import16resolve_relative
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys11audit_event
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os5build
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os7builtin
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os15initial_environ
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os14fsdecode_osstr
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os16stat_result_type
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject8new_user
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject14validate_bases
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject10solid_base
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os10builtin_kw
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE10initializeNCINvB2_11get_or_initNCNvNtNtBX_6stdlib2os22walk_symlinks_sentinel0E0zEBX_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvNtNtB1K_6stdlib2os22walk_symlinks_sentinel0E0zE0E0B1K_
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCNvMNtBa_9lazy_lockINtB1a_8LazyLockNtNtCs2D882usVKTN_10weavepy_vm8gc_trace7GcStateE5force0E0B1J_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState5track
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRyECs2D882usVKTN_10weavepy_vm
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockTFG_RL0_NtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEbFG0_RL1_B11_QL0_DG_INtNtNtCs8Mbv00yxnRz_4core3ops8function5FnMutTRL0_B11_EEp6OutputuEL0_EuEE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEB15_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys5build
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs2D882usVKTN_10weavepy_vm
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys18version_info_value
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os15struct_seq_type
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys22disallow_instantiation
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys7builtin
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort12sort4_stableReNvYB19_NtNtBa_3cmp10PartialOrd2ltECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry15register_module
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB1A_11Interpreter15locals_fast_off0E0zEB1A_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtCs8Mbv00yxnRz_4core6option6OptionNtCs2D882usVKTN_10weavepy_vm11InterpreterEEE10initializeNCINvB1a_11get_or_initNCNvNtB3M_13vm_singletons9seed_slot0E0zE0E0B3M_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm13builtin_types12BuiltinTypesEEEE10initializeNCINvB1a_11get_or_initNCNvNtB4m_13vm_singletons15seed_types_slot0E0zE0E0B4m_
-__RNvCs2D882usVKTN_10weavepy_vm22effective_ctype_locale
-__RNvMs0_NtCs2D882usVKTN_10weavepy_vm3gilNtB5_8GilState3new
-__RNvMs0_NtCs2D882usVKTN_10weavepy_vm3gilNtB5_8GilState7acquire
-__RNvNtCs2D882usVKTN_10weavepy_vm3gil14push_gil_guard
-__RNvNtCs2D882usVKTN_10weavepy_vm6stdlib12register_all
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache16register_builtin
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons23publish_interpreter_ptr
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache15builtin_factory
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache13frozen_source
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16push_frame_shell
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm6object10FrameShellEE8grow_oneB17_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier29try_enter
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier221jit_enabled_by_config
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellNtNtCs2D882usVKTN_10weavepy_vm5tier28JitStateEuE16get_or_init_slowNvNvB1R_3JIT27___rust_std_internal_init_fnEB1T_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter30run_until_yield_or_return_impl
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RNvXNtCs2D882usVKTN_10weavepy_vm6importNtB2_11ModuleCacheNtNtCs8Mbv00yxnRz_4core7default7Default7default
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys17venv_cfg_contents
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys13venv_cfg_path
-__RNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree11program_exe
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtB7_4path7PathBufEE10initializeNCINvB2_11get_or_initNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree7resolveE0zEB2s_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtBc_4path7PathBufEE10initializeNCINvB1a_11get_or_initNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree7resolveE0zE0E0B3g_
-__RINvMs16_NtCs82bWklYMk3w_3std4pathNtB7_4Path4joinNtNtCshxvaOLs88l5_5alloc6string6StringECs2D882usVKTN_10weavepy_vm
-__RINvXNtNtCs4w2dA8fTJLO_10serde_json5value2deNtB5_5ValueNtNtCs2GFHAg14Vqm_10serde_core2de11Deserialize11deserializeQINtNtB7_2de12DeserializerNtNtB7_4read7StrReadEECs2D882usVKTN_10weavepy_vm
-__RINvNvXs9_NtCs4w2dA8fTJLO_10serde_json2deINtB8_9MapAccesspENtNtCs2GFHAg14Vqm_10serde_core2de9MapAccess13next_key_seed12has_next_keyNtNtBa_4read7StrReadECs2D882usVKTN_10weavepy_vm
-__RNvXs8_NtCs4w2dA8fTJLO_10serde_json4readNtB5_7StrReadNtB5_4Read9parse_str
-__RINvXs9_NtCs4w2dA8fTJLO_10serde_json2deINtB6_9MapAccessNtNtB8_4read7StrReadENtNtCs2GFHAg14Vqm_10serde_core2de9MapAccess15next_value_seedINtNtCs8Mbv00yxnRz_4core6marker11PhantomDataNtNtB8_5value5ValueEECs2D882usVKTN_10weavepy_vm
-__RNvMs3_NtCs4w2dA8fTJLO_10serde_json2deINtB5_12DeserializerNtNtB7_4read7StrReadE13parse_integerCs2D882usVKTN_10weavepy_vm
-__RNvMs3_NtCs4w2dA8fTJLO_10serde_json2deINtB5_12DeserializerNtNtB7_4read7StrReadE12parse_numberCs2D882usVKTN_10weavepy_vm
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs4w2dA8fTJLO_10serde_json
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs4w2dA8fTJLO_10serde_json5value5ValueECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs4w2dA8fTJLO_10serde_json5value5ValueECs2D882usVKTN_10weavepy_vm
-__RINvMNtCs8Mbv00yxnRz_4core3stre12trim_matchesNvMNtNtB5_4char7methodsc13is_whitespaceECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace13has_finalizer
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState10bump_count
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys16build_with_state
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtNtNtBe_11collections4hash3map7HashMapReTINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB1j_5types10TypeObjectERNtNtNtB1j_6stdlib2os15StructSeqLayoutEEEuE16get_or_init_slowNvNvB3H_19STRUCT_SEQ_REGISTRY27___rust_std_internal_init_fnEB1j_
-__RNvMNtCs2D882usVKTN_10weavepy_vm13tuple_storageNtB2_12TupleStorage8from_vec
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os19struct_seq_instance
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RINvNtNtCshxvaOLs88l5_5alloc3vec16in_place_collect18from_iter_in_placeINtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtB4_9into_iter8IntoIterReENvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB2v_6Object11from_staticEB34_EB2x_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm9hot_gates9env_flags8no_quiet0E0zE0E0B2o_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB2o_11Interpreter15locals_fast_off0E0zE0E0B2o_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtCs8Mbv00yxnRz_4core6option6OptionNtCs2D882usVKTN_10weavepy_vm11InterpreterEEE10initializeNCINvB2_11get_or_initNCNvNtB2Z_13vm_singletons9seed_slot0E0zEB2Z_
-__RNvMs19_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6PyFile10set_errors
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm3gil8GilStateEE10initializeNCINvB1a_11get_or_initNCNvB2g_14global_gil_ref0E0zE0E0B2i_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11import_path
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter8load_one
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache21is_third_party_facade
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache28find_source_shadowing_stdlib
-__RINvXsE_NtCs82bWklYMk3w_3std4pathNtB6_7PathBufINtNtNtNtCs8Mbv00yxnRz_4core4iter6traits7collect12FromIteratorReE9from_iterINtNtNtBR_3str4iter5SplitcEECs2D882usVKTN_10weavepy_vm
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache11search_dirs
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtCs82bWklYMk3w_3std4path7PathBufEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters10filter_map9FilterMapINtNtNtB2e_5slice4iter4IterNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectENCNvMs_NtB3C_6importNtB4o_11ModuleCache11search_dirs0EE9from_iterB3C_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCshxvaOLs88l5_5alloc3vec9into_iter8IntoIterNtNtCs82bWklYMk3w_3std4path7PathBufEECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree11module_path
-__RINvMs16_NtCs82bWklYMk3w_3std4pathNtB7_4Path4joinNtB7_7PathBufECs2D882usVKTN_10weavepy_vm
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtNtNtBe_11collections4hash3map7HashMapReNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectEEuE16get_or_init_slowNvNvNtB1j_17frozen_code_cache5CACHE27___rust_std_internal_init_fnEB1j_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_mod15load_from_bytes
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader4take
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader8note_ref
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE8grow_oneB1s_
-__RNvMNtNtCs2D882usVKTN_10weavepy_vm6stdlib13codecs_engineNtB2_6DecCtx3new
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object19str_from_codepoints
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib13codecs_engine6DecCtxEBH_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys11intern_name
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapNtNtCs2D882usVKTN_10weavepy_vm12shared_value9SharedStruNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertBR_
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object13as_bytes_view
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_mod9string_of
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_mod18tuple_to_constants
-__RNvMs6_NtCs6MP6LIxl4z2_16weavepy_compiler12cpython_codeNtB5_7SlotMap10from_kinds
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code10DecodedRawE8grow_oneBQ_
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code19decode_instructions
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code14unit_index_map
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjjEE14reserve_rehashNCINvNtB8_3map11make_hasherjjNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs6MP6LIxl4z2_16weavepy_compiler
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code15raw_first_instr
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code8SlotKindE8grow_oneBQ_
-__RNvMsd_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_12PyMemoryView7release
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object16MemoryViewBufferEBF_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_15memoryview_castEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_14memoryview_hexEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_18memoryview_tobytesEB4_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object10PyPropertyE9drop_slowBK_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_16property_init_kwEB4_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object5RangeE9drop_slowBK_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object7PySliceE9drop_slowBK_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_17staticmethod_callEB4_
-__RINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB2_13str_format_kwEB4_
-__RNvXsg_NtCs3fRkGY2FHAU_9hashbrown3rawINtB5_8RawTableTNtNtCshxvaOLs88l5_5alloc6string6StringINtNtBT_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB1M_
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvNtB1K_13vm_singletons8ellipsis0E0zE0E0B1K_
-__RINvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB7_6Object8from_strRNtNtCshxvaOLs88l5_5alloc6string6StringEB9_
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree7resolve
-__RNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree13landmark_walk
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvMsi_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB5_8BTreeMapNtNtBb_6string6StringNtNtCs4w2dA8fTJLO_10serde_json5value5ValueE6insertCs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs4w2dA8fTJLO_10serde_json5value5ValueE8grow_oneBQ_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvXs_NtNtCs4w2dA8fTJLO_10serde_json5value5indexeNtB4_5Index10index_into
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs4w2dA8fTJLO_10serde_json5value5ValueECs2D882usVKTN_10weavepy_vm
-__RNvMsz_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB5_8IntoIterNtNtBb_6string6StringNtNtCs4w2dA8fTJLO_10serde_json5value5ValueE10dying_nextCs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE8grow_oneBQ_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTReFG_RL0_NtNtCs2D882usVKTN_10weavepy_vm6import11ModuleCacheEINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB13_6object8PyModuleEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_BS_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB13_
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache6insert
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace5track
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState3new
-__RNvNtCs2D882usVKTN_10weavepy_vm16weakref_registry5id_of
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTyINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm8gc_trace13TrackedHandleEEE14reserve_rehashNCINvNtB8_3map11make_hasheryBR_INtNtCs8Mbv00yxnRz_4core4hash18BuildHasherDefaultNtB1r_13GcIndexHasherEE0EB1t_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm8gc_trace13TrackedHandleEE8grow_oneB17_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once9call_onceNCNvXs1_Cs2D882usVKTN_10weavepy_vmNtB16_11InterpreterNtNtCs8Mbv00yxnRz_4core7default7Default7defaults0_0E0B16_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockTFG_RL0_NtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEbFG0_RL1_B1O_QL0_DG_INtNtNtCs8Mbv00yxnRz_4core3ops8function5FnMutTRL0_B1O_EEp6OutputuEL0_EuEE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B1S_
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB10_
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE12shift_removeBO_EBS_
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE17shift_remove_fullBO_EBS_
-__RNvMs_NtCsblfCQaHOzq5_8indexmap5innerINtB4_4CoreNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBM_6ObjectE19shift_remove_finishBO_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTReTINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectERNtNtNtB1v_6stdlib2os15StructSeqLayoutEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_BS_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1v_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEBX_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B1K_
-__RINvMNtCs2D882usVKTN_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj3_EB5_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RINvMs19_NtCs2D882usVKTN_10weavepy_vm6objectNtB7_6PyFile3newReBW_EB9_
-__RNvMs18_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6IoKind9from_mode
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib14sys_monitoring5build
-__RNvNtCs2D882usVKTN_10weavepy_vm5trace20monitoring_sentinels
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5trace20monitoring_sentinels0B5_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os22struct_seq_type_layout
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortReNvYBT_NtNtB8_3cmp10PartialOrd2ltECs2D882usVKTN_10weavepy_vm
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable9quicksort9quicksortReNvYB17_NtNtBa_3cmp10PartialOrd2ltECs2D882usVKTN_10weavepy_vm
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared5pivot11median3_recReNvYB14_NtNtBa_3cmp10PartialOrd2ltECs2D882usVKTN_10weavepy_vm
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm9hot_gates9env_flags8no_quiet0E0zEB1A_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm9hot_gates9env_flags8no_burst0E0zEB1A_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm9hot_gates9env_flags8no_burst0E0zE0E0B2o_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm9hot_gates9env_flags10dbg_sample0E0zEB1A_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm9hot_gates9env_flags10dbg_sample0E0zE0E0B2o_
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons24publish_interpreter_seed
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15fork_for_thread
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm13builtin_types12BuiltinTypesEEEE10initializeNCINvB2_11get_or_initNCNvNtB3z_13vm_singletons15seed_types_slot0E0zEB3z_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17apply_run_options
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm3gil8GilStateEE10initializeNCINvB2_11get_or_initNCNvB1t_14global_gil_ref0E0zEB1v_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2D882usVKTN_10weavepy_vm3gil8GilGuardE8grow_oneBQ_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecONtCs2D882usVKTN_10weavepy_vm11InterpreterE8grow_oneBP_
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache3get
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache19initializing_holder
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache17try_begin_loading
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapNtNtCshxvaOLs88l5_5alloc6string6StringyNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtCshxvaOLs88l5_5alloc6string6StringyEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_yNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache13live_sys_path
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RNvNtCs2D882usVKTN_10weavepy_vm6import19is_test_frozen_name
-__RNvNtCs2D882usVKTN_10weavepy_vm17frozen_code_cache3get
-__RNvXsb_NtCs6MP6LIxl4z2_16weavepy_compiler8bytecodeNtB5_10CacheTableNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvNtCs2D882usVKTN_10weavepy_vm17frozen_code_cache8get_disk
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtB7_4path7PathBufEE10initializeNCINvB2_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm17frozen_code_cache8disk_dir0E0zEB2u_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtBc_4path7PathBufEE10initializeNCINvB1a_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm17frozen_code_cache8disk_dir0E0zE0E0B3i_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader10read_value
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader9read_byte
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader9read_code
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader8read_int
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10codecs_mod16decode_bytes_obj
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtNtNtBa_11collections4hash3set7HashSetNtNtBZ_12shared_value9SharedStrEEE4withNCNvNtNtBZ_6stdlib3sys11intern_name0NtNtBZ_6object6ObjectEBZ_
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtNtNtBe_11collections4hash3set7HashSetNtNtB1j_12shared_value9SharedStrEEuE16get_or_init_slowNvNvNtNtB1j_6stdlib3sys11INTERN_POOL27___rust_std_internal_init_fnEB1j_
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRNtNtCs2D882usVKTN_10weavepy_vm12shared_value9SharedStrEB1J_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtCs2D882usVKTN_10weavepy_vm12shared_value9SharedStruEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBU_
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader8fill_ref
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_mod8bytes_of
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_mod16tuple_of_strings
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtB2o_5slice4iter4IterNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectENvB3y_25object_to_constant_publicEE9from_iterB3y_
-__RNvCs2D882usVKTN_10weavepy_vm18object_to_constant
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code11decode_full
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler12cpython_code11decode_raws
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_mod13MarshalReaderEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB1y_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB12_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtBG_13tuple_storage12TupleStorageEEBG_
-__RNvNtCs2D882usVKTN_10weavepy_vm7pycache16own_decoded_code
-__RNvNtCs2D882usVKTN_10weavepy_vm7pycache17rewrite_filenames
-__RNvNvNtCs2D882usVKTN_10weavepy_vm7pycache17rewrite_filenames4walk
-__RNvNtCs2D882usVKTN_10weavepy_vm17frozen_code_cache6insert
-__RNvMs_NtCshxvaOLs88l5_5alloc3vecINtB4_3VechE16into_boxed_sliceCs2D882usVKTN_10weavepy_vm
-__RNvXsr_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_10CodeObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvXsP_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8ConstantNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtB7_6string6StringENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCs2D882usVKTN_10weavepy_vm
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecmENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCs2D882usVKTN_10weavepy_vm
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VechENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTReNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_BS_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19run_frozen_compiled
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20build_module_globals
-__RNvNtCs2D882usVKTN_10weavepy_vm7pycache14cache_path_for
-__RNvXsq_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache18begin_initializing
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter10make_frame
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter23pooled_locals_from_args
-__RNvMs1_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE6resizeBJ_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15recycle_scratch
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object11empty_cells
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtB1L_3vec3VecIB1H_INtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtB2F_6object6ObjectEEEEE10initializeNCINvB1a_11get_or_initNCNvB3k_15empty_cells_ref0E0zE0E0B2F_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_4sync3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtB1u_6object6ObjectEEEEB1u_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter9run_frame
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter25run_until_yield_or_return
-__RNvCsqZnmI3D1IW_7stacker15remaining_stack
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell4CellINtNtB1j_6option6OptionjEEzE16get_or_init_slowNvNvCsqZnmI3D1IW_7stacker11STACK_LIMIT27___rust_std_internal_init_fnEB2A_
-__RNvCs6MP6LIxl4z2_16weavepy_compiler20comp_clause_is_async
-__RNvCs6MP6LIxl4z2_16weavepy_compiler19expr_contains_await
-__RNvCs6MP6LIxl4z2_16weavepy_compiler24expr_contains_async_comp
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler21compile_comprehension
-__RNvCs6MP6LIxl4z2_16weavepy_compiler22check_comp_walrus_nest
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtCs6MP6LIxl4z2_16weavepy_compiler15CompWalrusScopeE8grow_oneBO_
-__RNvNvCs6MP6LIxl4z2_16weavepy_compiler25reject_walrus_in_iterable4scan
-__RNvNvCs6MP6LIxl4z2_16weavepy_compiler22check_comp_walrus_nest5visit
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs6MP6LIxl4z2_16weavepy_compiler9flowgraph11HandlerInfoE8grow_oneBQ_
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler20comp_inline_eligible
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler15CompWalrusScopeEEB1a_
-__RNvCs6MP6LIxl4z2_16weavepy_compiler17compile_comp_body
-__RNvCs6MP6LIxl4z2_16weavepy_compiler19stamp_comp_elt_span
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler21note_identifier_const
-__RNvMs0_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg15make_load_const
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler7ast_opt18eval_unaryop_const
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler9flowgraph15to_ast_constant
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast8ConstantECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler9flowgraphNtB5_3Cfg11jump_thread
-__RINvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_stmt_inners_0EB6_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast7KeywordE8grow_oneBQ_
-__RNvXsP_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8ConstantNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvNtCs2D882usVKTN_10weavepy_vm3gil15maybe_yield_gil
-__RNvNtCs2D882usVKTN_10weavepy_vm10ext_loader24current_extension_loader
-__RNvNtCsl2apZsWCN6y_12weavepy_capi6loader17vm_load_extension
-__RNvNtCsl2apZsWCN6y_12weavepy_capi5embed14inittab_lookup
-__RINvMNtNtNtCs82bWklYMk3w_3std3sys4sync8once_boxINtB3_7OnceBoxNtNtNtNtNtB7_3pal4unix4sync5mutex5MutexE10initializeNCNvMNtNtB5_5mutex7pthreadNtB1S_5Mutex3get0ECsl2apZsWCN6y_12weavepy_capi
-__RNvNtCs2D882usVKTN_10weavepy_vm10ext_loader18take_explicit_path
-__RNvNtCsl2apZsWCN6y_12weavepy_capi6loader22find_extension_on_path
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECsl2apZsWCN6y_12weavepy_capi
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRNtNtNtB9_3ffi6os_str8OsStringECs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtNtCs82bWklYMk3w_3std3ffi6os_str8OsStringuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtNtNtBW_4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler28compile_deferred_annotations
-__RNvNvCs6MP6LIxl4z2_16weavepy_compiler24collect_self_attr_stores6target
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler13closure_index
-__RNvCs6MP6LIxl4z2_16weavepy_compiler24collect_self_attr_stores
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10os_process8register
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEE10initializeNCINvB1a_11get_or_initNCNvNtNtB2i_6stdlib2os14path_like_type0E0zE0E0B2i_
-__RINvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB7_6Object12new_set_fromINtNtCshxvaOLs88l5_5alloc3vec3VecBG_EEB9_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object13identity_hash
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecyE8grow_oneCsfGIcTr55AEy_10num_bigint
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsfGIcTr55AEy_10num_bigint
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache19builtin_static_name
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter12bind_os_path
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib9errno_mod5build
-__RNvNtNtCsfGIcTr55AEy_10num_bigint7biguint5shift12biguint_shl2
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNvMs2_Cs2D882usVKTN_10weavepy_vmNtBJ_11Interpreter8load_one12LoadingGuardEBJ_
-__RNvXNvMs2_Cs2D882usVKTN_10weavepy_vmNtB8_11Interpreter8load_oneNtB2_12LoadingGuardNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24lookup_global_or_builtin
-__RNvMsh_NtCsfGIcTr55AEy_10num_bigint7biguintNtB5_7BigUint10normalized
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecyENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCs2D882usVKTN_10weavepy_vm
-__RNvXNtNtCsfGIcTr55AEy_10num_bigint7biguint8additionNtB4_7BigUintINtNtNtCs8Mbv00yxnRz_4core3ops5arith3AddRBN_E3add
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object15int_from_bigint
-__RINvMs2_Cs2D882usVKTN_10weavepy_vmNtB6_11Interpreter17alloc_tuple_arrayKj2_EB6_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15leaf_burst_slow
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtB6_5drain5DrainB13_EE9from_iterB17_
-__RINvMsj_NtCshxvaOLs88l5_5alloc3vecINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE14extend_trustedINtNtB6_5drain5DrainBG_EEBK_
-__RNvXs5_NtNtCshxvaOLs88l5_5alloc3vec5drainINtB5_5DrainNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropBT_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11alloc_tuple
-__RINvMs2_Cs2D882usVKTN_10weavepy_vmNtB6_11Interpreter17alloc_tuple_arrayKj3_EB6_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtBG_13tuple_storage12TupleStorageEEBG_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_5boxed3BoxNtCs2D882usVKTN_10weavepy_vm9InlineActEEEB1s_
-__RINvMNtCs2D882usVKTN_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj2_EB5_
-__RINvMs2_Cs2D882usVKTN_10weavepy_vmNtB6_11Interpreter17alloc_tuple_arrayKj1_EB6_
-__RINvMNtCs2D882usVKTN_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj1_EB5_
-__RNvMsf_NtNtCs8Mbv00yxnRz_4core3str4iterINtB5_13SplitInternalcE9next_backCsl2apZsWCN6y_12weavepy_capi
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtB4_2io5error5ErrorECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionTNtNtCs82bWklYMk3w_3std4path7PathBufINtNtCshxvaOLs88l5_5alloc2rc2RcINtNtNtNtB12_11collections4hash3set7HashSetNtNtNtB12_3ffi6os_str8OsStringEEEEECsl2apZsWCN6y_12weavepy_capi
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapNtNtBe_4path7PathBufTNtNtCs2D882usVKTN_10weavepy_vm6import8DirStampINtNtCshxvaOLs88l5_5alloc2rc2RcINtNtB1U_3set7HashSetNtNtNtBe_3ffi6os_str8OsStringEEEEEuE16get_or_init_slowNvNvB2S_12DIR_LISTINGS27___rust_std_internal_init_fnEB2U_
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRNtNtB9_4path7PathBufECs2D882usVKTN_10weavepy_vm
-__RINvXs1h_NtCs82bWklYMk3w_3std4pathNtB7_4PathNtNtCs8Mbv00yxnRz_4core4hash4Hash4hashNtNtNtB9_4hash6random13DefaultHasherECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm6import11dir_listing
-__RNvXs2_NtNtCs82bWklYMk3w_3std4hash6randomNtB5_13DefaultHasherNtNtCs8Mbv00yxnRz_4core4hash6Hasher5write
-__RNvXs1i_NtCs8Mbv00yxnRz_4core3fmtRReNtB6_7Display3fmtCsl2apZsWCN6y_12weavepy_capi
-__RINvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB6_7HashMapNtNtNtCs82bWklYMk3w_3std3ffi6os_str8OsStringuNtNtNtBU_4hash6random11RandomStateE12contains_keyNtBQ_5OsStrECsl2apZsWCN6y_12weavepy_capi
-__RNvXs3_NtNtCs8Mbv00yxnRz_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCsl2apZsWCN6y_12weavepy_capi
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache11find_source
-__RINvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB6_7HashMapNtNtNtCs82bWklYMk3w_3std3ffi6os_str8OsStringuNtNtNtBU_4hash6random11RandomStateE12contains_keyNtBQ_5OsStrECs2D882usVKTN_10weavepy_vm
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache22find_namespace_package
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs82bWklYMk3w_3std4path7PathBufEECs2D882usVKTN_10weavepy_vm
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20try_meta_path_import
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapNtNtCshxvaOLs88l5_5alloc6string6StringuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCs2D882usVKTN_10weavepy_vm
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache16end_initializing
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter4call
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier222try_call_native_direct
-__RNvXs6_NtCs2D882usVKTN_10weavepy_vm6objectINtB5_9FrameSlotINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtBZ_3vec3VecIBV_INtNtB7_4sync7GilCellNtB5_6ObjectEEEEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB7_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm9StampSlotEE10initializeNCINvB2_11get_or_initNCNvB1v_15code_stamp_slot0E0zEB1v_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter23specialized_load_global
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize30attempt_specialize_load_global
-__RINvMNtNtCs8Mbv00yxnRz_4core4cell4onceINtB3_8OnceCellNtCs2D882usVKTN_10weavepy_vm7LeafFnsE8try_initNCINvB2_11get_or_initNCNvMs2_BR_NtBR_11Interpreter8leaf_fns0E0zEBR_
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjNtCs2D882usVKTN_10weavepy_vm8LeafKindNtNtBQ_8fasthash13FxBuildHasherE6insertBQ_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtCs2D882usVKTN_10weavepy_vm8LeafRecvReINtNtB7_4sync3ArcNtNtBP_6object9BuiltinFnEEE8grow_oneBP_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6result6ResultNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtNtB11_5error12RuntimeErrorEEB11_
-__RNvMs3_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE11extend_withBJ_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier221try_call_native_bound
-__RNvXs_NtCs2D882usVKTN_10weavepy_vm10specializeNtNtB6_6object8DictDataNtB4_11DictDataExt16index_of_key_str
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace18note_dropped_marks
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins14str_rpartitionINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14str_rpartition
-__RINvNtCs2D882usVKTN_10weavepy_vm13builtin_types14make_exceptionNtNtCshxvaOLs88l5_5alloc6string6StringEB4_
-__RNvMNtCs2D882usVKTN_10weavepy_vm13builtin_typesNtB2_12BuiltinTypes7by_name
-__RINvNtCs2D882usVKTN_10weavepy_vm13builtin_types25make_exception_with_classNtNtCshxvaOLs88l5_5alloc6string6StringEB4_
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types19is_subclass_by_name
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object6to_str
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RNvMse_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10PyInstance8slot_set
-__RNvMse_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10PyInstance17ensure_gc_tracked
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5types8SlotDataEBF_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBP_6ObjectEE8grow_oneBR_
-__RNvNtCs2D882usVKTN_10weavepy_vm5error18set_exception_attr
-__RNvMse_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10PyInstance8slot_get
-__RNvMsd_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_11SlotStorage3get
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter23attach_implicit_context
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs8Mbv00yxnRz_4core4cell4CellTymEEE4withNCNvNtCs2D882usVKTN_10weavepy_vm5trace28monitoring_union_mask_cached0mEB1J_
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtBZ_5trace15MonitoringToolsEE4withNCNvB1E_21monitoring_union_mask0mEBZ_
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtB1j_5trace15MonitoringToolsEuE16get_or_init_slowNvNvB1Y_16MONITORING_TOOLS27___rust_std_internal_init_fnEB1j_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16handle_exception
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2D882usVKTN_10weavepy_vm5error14TracebackEntryE8grow_oneBQ_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object20materialize_stack_at
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_6object7PyFrameEEE6borrowB7_
-__RNvMs8_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10FrameShell11materialize
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync4WeakNtNtB7_6object11PyGeneratorEEE6borrowB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_6object7PyFrameEEE10borrow_mutB7_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm6object10FrameShellEEEB1t_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5error11PyExceptionEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm5error14TracebackEntryEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5error14TracebackEntryEBF_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19ensure_top_py_frame
-__RNvNtCs2D882usVKTN_10weavepy_vm6object21materialize_stack_top
-__RNvXs7_NtCs2D882usVKTN_10weavepy_vm5errorNtB5_11PyExceptionNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCs2D882usVKTN_10weavepy_vm5error14TracebackEntryENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState16auto_collect_due
-__RINvMsj_NtCshxvaOLs88l5_5alloc3vecINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE14extend_trustedINtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters6cloned6ClonedINtNtNtB1O_5slice4iter4IterBG_EEEBK_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15pop_frame_shell
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19recycle_frame_shell
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter25reap_frame_locals_on_exit
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20recycle_frame_allocs
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE8grow_oneB16_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_4sync3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtB7_3vec3VecNtNtB18_6object6ObjectEEEE8grow_oneB18_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs2D882usVKTN_10weavepy_vm5FrameEBD_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTmNtNtCs2D882usVKTN_10weavepy_vm5error11PyExceptionEEEB1e_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm5error11PyExceptionEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtNtCs2D882usVKTN_10weavepy_vm5tier216NativeActivationEEEB1A_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17call_python_owned
-__RNvMsU_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyFunction4code
-__RNvXs6_NtCs2D882usVKTN_10weavepy_vm6objectINtB5_9FrameSlotINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtB7_4sync7GilCellINtNtBZ_3vec3VecNtB5_6ObjectEEEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB7_
-__RNvXs6_NtCs2D882usVKTN_10weavepy_vm6objectINtB5_9FrameSlotINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtB7_4sync7GilCellNtB5_8DictDataEEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB7_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16leaf_core_helper
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14leaf_core_attr
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm9StampSlotEE10initializeNCINvB1a_11get_or_initNCNvB2i_15code_stamp_slot0E0zE0E0B2i_
-__RNvMs8_NtCs6MP6LIxl4z2_16weavepy_compiler8bytecodeNtB5_10CacheTable8allocate
-__RNvMs4_NtCs6MP6LIxl4z2_16weavepy_compiler8bytecodeNtB5_9CacheSlot3new
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14leaf_core_call
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjNtCs2D882usVKTN_10weavepy_vm8LeafKindEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtBT_8fasthash13FxBuildHasherE0EBT_
-__RNvCs2D882usVKTN_10weavepy_vm17object_dunder_new
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18load_method_ic_hit
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21specialized_load_attr
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter9load_attr
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15load_attr_inner
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter10maybe_bind
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize28attempt_specialize_load_attr
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15try_inline_call
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13try_lean_call
-__RNvMs_NtCshxvaOLs88l5_5alloc3vecINtB4_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE6removeBI_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13dispatch_call
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14pooled_scratch
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize23attempt_specialize_call
-__RNvCs2D882usVKTN_10weavepy_vm20builtin_needs_interp
-__RNvCs2D882usVKTN_10weavepy_vm21code_pure_leaf_decide
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18call_from_bytecode
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter9core_call
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm8CallSlotEE10initializeNCINvB2_11get_or_initNCNvB1v_14code_call_slot0E0zEB1v_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15call_c_profiled
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm8CallSlotEE10initializeNCINvB1a_11get_or_initNCNvB2i_14code_call_slot0E0zE0E0B2i_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins10str_result
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18reap_call_receiver
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object11BoundMethodE9drop_slowBK_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14reap_call_args
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object9make_iter
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter22iter_backing_list_dead
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE6borrowB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE10borrow_mutB7_
-__RNvMs1e_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_10PyIterator18next_value_checked
-__RNvMs1e_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_10PyIterator10next_value
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17try_lean_property
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14leaf_type_call
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtBG_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB1y_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState18maybe_auto_collect
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace7untrack
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm10MethodSlotEE10initializeNCINvB1a_11get_or_initNCNvB2i_16code_method_slot0E0zE0E0B2i_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14leaf_call_kind
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueTyNtCs2D882usVKTN_10weavepy_vm12MethodSlotFnEEBF_
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object3len
-__RNvNtCs2D882usVKTN_10weavepy_vm6object12str_char_len
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalEBO_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState10is_tracked
-__RNvNtCs2D882usVKTN_10weavepy_vm16weakref_registry18strong_clone_count
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17try_lean_for_iter
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20specialized_for_iter
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize27attempt_specialize_for_iter
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17reap_iter_backing
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace16strong_count_for
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject14is_subclass_of
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19prompt_reap_dropped
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21object_is_finalizable
-__RNvNtCs2D882usVKTN_10weavepy_vm16weakref_registry9count_for
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16is_refcount_dead
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm9hot_gates9env_flags10reap_trace0E0zEB1A_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState22rebuild_tracked_filter
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter23activate_thread_handles
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons24current_worker_thread_id
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtNtNtB7_11collections4hash3map7HashMapyyEEE10initializeNCINvB2_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons10worker_map0E0zEB3J_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib16faulthandler_mod17note_thread_start
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib16faulthandler_mod16RegisteredThreadE8grow_oneBS_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2D882usVKTN_10weavepy_vm13vm_singletons13ThreadHandlesE8grow_oneBQ_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object23collect_small_int_range
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm8gc_trace13TrackedHandleE9drop_slowBK_
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters6cloned6ClonedINtNtNtB2p_5slice4iter4IterB13_EEE9from_iterB17_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_lenINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB1j_6option6OptionTjNtNtCs2D882usVKTN_10weavepy_vm12shared_value9SharedStrjEEEEuE16get_or_init_slowNvNvNtB2P_6object13STR_LEN_CACHE27___rust_std_internal_init_fnEB2P_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins18new_str_split_list
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvMsj_NtCs2D882usVKTN_10weavepy_vm12shared_valueNtB5_9SharedStr6concat
-__RNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB5_8JitState13note_backedge
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17type_call_default
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11instantiate
-__RNvCs2D882usVKTN_10weavepy_vm20object_needs_vm_iter
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtNtNtBc_11collections4hash3map7HashMapyyEEE10initializeNCINvB1a_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons10worker_map0E0zE0E0B4x_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16collect_iterable
-__RNvXs0_NtNtCshxvaOLs88l5_5alloc3vec11spec_extendINtB7_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB5_10SpecExtendBU_INtNtB7_9into_iter8IntoIterBU_EE11spec_extendBY_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState10untrack_id
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace14remove_suspect
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14leaf_fused_len
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17leaf_builtin_call
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_arity
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject17metaclass_or_type
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins6b_typeINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace24note_dropped_marks_other
-__RINvXs2J_NtNtCs8Mbv00yxnRz_4core5slice4iterINtB7_4IterNtNtCs6MP6LIxl4z2_16weavepy_compiler8bytecode11InstructionENtNtNtNtBb_4iter6traits8iterator8Iterator3allNCNvCs2D882usVKTN_10weavepy_vm21code_pure_leaf_decide0EB2A_
-__RNvMsT_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyFunction14lean_cells_ref
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter10lean_frame
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14run_frame_lean
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19run_frame_lean_impl
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15flush_lean_cold
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15push_lean_shell
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17retire_lean_frame
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24specialized_store_subscr
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins15ensure_dict_key
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins15ensure_hashable
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11dict_insert
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons23current_interpreter_ptr
-__RNvNtCs2D882usVKTN_10weavepy_vm6object15dict_watch_bump
-__RNvNtCs2D882usVKTN_10weavepy_vm6object19dict_mutation_event
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6result6ResultINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectENtNtB1n_5error12RuntimeErrorEEB1n_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18reap_dead_subgraph
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm13vm_singletons13ThreadHandlesEBF_
-__RNvXs0_NtCs2D882usVKTN_10weavepy_vm13vm_singletonsNtB5_16InterpreterGuardNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20emit_escape_warnings
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13run_module_as
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter22run_pending_finalizers
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm3gil8GilGuardEEB11_
-__RNvXsu_Cs7Bv8z6zqnmV_4itoayNtB5_8Unsigned3fmt
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject23instances_need_finalize
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtCs82bWklYMk3w_3std4path7PathBufTNtNtCs2D882usVKTN_10weavepy_vm6import8DirStampINtNtCshxvaOLs88l5_5alloc2rc2RcINtNtNtNtBU_11collections4hash3set7HashSetNtNtNtBU_3ffi6os_str8OsStringEEEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1p_NtNtNtBU_4hash6random11RandomStateE0EB1u_
-__RNvXs1i_NtCs8Mbv00yxnRz_4core3fmtReNtB6_7Display3fmtCsl2apZsWCN6y_12weavepy_capi
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTmNtNtCs2D882usVKTN_10weavepy_vm5error11PyExceptionEE8grow_oneBS_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2D882usVKTN_10weavepy_vm5error11PyExceptionE8grow_oneBQ_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17exception_matches
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types20instance_is_subclass
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19exc_has_finalizable
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace15traverse_object
-__RNvYNCNCNvXs1_Cs2D882usVKTN_10weavepy_vmNtBc_11InterpreterNtNtCs8Mbv00yxnRz_4core7default7Default7defaults0_00INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
-__RNvYNCNCNvXs1_Cs2D882usVKTN_10weavepy_vmNtBc_11InterpreterNtNtCs8Mbv00yxnRz_4core7default7Default7defaults0_0s_0INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
-__RNvYNCNCNvXs1_Cs2D882usVKTN_10weavepy_vmNtBc_11InterpreterNtNtCs8Mbv00yxnRz_4core7default7Default7defaults0_0s1_0INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
-__RNCNvNvMs2_Cs2D882usVKTN_10weavepy_vmNtB9_11Interpreter21anchors_tracked_child4walk0B9_
-__RNvCs2D882usVKTN_10weavepy_vm17py_frame_traverse
-__RNvXsf_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10PyInstanceNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtB7_3vec3VecIBx_INtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtB15_6object6ObjectEEEE9drop_slowB15_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtBL_6object6ObjectEE9drop_slowBL_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCNvMNtBa_9lazy_lockINtB1a_8LazyLockNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry15WeakRefRegistryE5force0E0B1J_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16pop_jump_if_step
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object9is_truthy
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter12pop_top_step
-__RNvMse_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10PyInstance18dies_by_plain_drop
-__RNvMsd_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_11SlotStorage4iter
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter19exc_has_finalizable0B7_
-__RNvYNCNCNvXs1_Cs2D882usVKTN_10weavepy_vmNtBc_11InterpreterNtNtCs8Mbv00yxnRz_4core7default7Default7defaults0_0s0_0INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
-__RNvCs2D882usVKTN_10weavepy_vm21py_traceback_traverse
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10PyInstanceE9drop_slowBK_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object11PyTracebackE9drop_slowBK_
-__RNvXs9_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_11PyTracebackNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_7PyFrame14refresh_locals
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object7PyFrameE9drop_slowBK_
-__RNvXs1_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_7PyFrameNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object10FrameShellE9drop_slowBK_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_splitE9drop_slowBK_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNvNtCs2D882usVKTN_10weavepy_vm8builtins14str_replace_kwE9drop_slowBK_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit6engine9JitEngineEECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm6object16is_function_slot
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize29attempt_specialize_store_attr
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry3new
-__RNvCs2D882usVKTN_10weavepy_vm16metaclass_method
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_dir
-__RNvMsi_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB5_8BTreeMapNtNtBb_6string6StringNtNtB7_7set_val9SetValZSTE6insertCs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14str_startswith
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11list_appendINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier213try_enter_osr
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm8gc_trace12note_suspect0E0zE0E0B2m_
-__RNvYNCNvNtCs2D882usVKTN_10weavepy_vm8gc_trace8SUSPECTS0INtNtNtCs8Mbv00yxnRz_4core3ops8function6FnOnceuE9call_onceB8_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13contains_full
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_setINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyNtNtCs2D882usVKTN_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_13key_cmp_scopebNCNvMs2_BY_NtBY_11Interpreter4stepsA_0E0INtNtCs8Mbv00yxnRz_4core6result6ResultbNtNtBY_5error12RuntimeErrorEEBY_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvCs2D882usVKTN_10weavepy_vm5tsdbg0E0zEB1w_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13sort_with_key
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable14driftsort_mainNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNCINvMNtCshxvaOLs88l5_5alloc5sliceSBZ_7sort_byNCNvMs2_B13_NtB13_11Interpreter13sort_with_keys3_0E0INtNtB1P_3vec3VecBZ_EEB13_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable9quicksort9quicksortNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNCINvMNtCshxvaOLs88l5_5alloc5sliceSB15_7sort_byNCNvMs2_B19_NtB19_11Interpreter13sort_with_keys3_0E0EB19_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins8dict_get
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16module_shadowing
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11import_from
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyNtNtCs2D882usVKTN_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_20with_key_eq_deferredINtNtCs8Mbv00yxnRz_4core6result6ResultINtNtB2u_6option6OptionIB32_NtBW_6ObjectEENtNtBY_5error12RuntimeErrorENCNvMs2_BY_NtBY_11Interpreter17leaf_builtin_calls1_0E0TB2p_bEEBY_
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE12get_index_ofNtBQ_12StrKeyHashedEBS_
-__RINvXNtNtCshxvaOLs88l5_5alloc3vec14spec_from_elemNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEBP_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter9make_iter
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11b_frozensetINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject17lookup_with_owner
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object8DictDataEBF_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry14collect_strong
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter25specialized_binary_subscr
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins10b_reversed
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object9as_bigint
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15weave_frame_mod5build
-__RNvYFG_RL0_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtNtCs8Mbv00yxnRz_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
-__RNvXNtCs8Mbv00yxnRz_4core3anyNtCs2D882usVKTN_10weavepy_vm5FrameNtB2_3Any7type_idBt_
-__RNvMs11_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_11PyGenerator11is_finished
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState10fin_insert
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16gen_method_close
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState13fin_make_cold
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm8gc_trace11fin_no_cold0E0zEB1y_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB2o_7GcState28reap_dead_finalizable_locked0E0zE0E0B2q_
-__RNCNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB7_7GcState28reap_dead_finalizable_lockeds0_0B9_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm12type_surface24install_type_dict_extras0B5_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14sync_py_locals
-__RNvNtCs2D882usVKTN_10weavepy_vm16weakref_registry15queue_callbacks
-__RNCNvNtCs2D882usVKTN_10weavepy_vm13builtin_types16make_default_new0B5_
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types10object_new
-__RINvNtNtCs8Mbv00yxnRz_4core4iter8adapters11try_processINtNtB2_3map3MapQDNtNtNtB4_6traits8iterator8Iteratorp4ItemRNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEL_NCNCNvMs2_B1R_NtB1R_11Interpreter11import_star00ENtNtCshxvaOLs88l5_5alloc6string6StringINtNtB6_6result6ResultNtNtB6_7convert10InfallibleNtNtB1R_5error12RuntimeErrorENCINvXso_B40_IB3Y_INtNtB3p_3vec3VecB3l_EB4K_EINtNtB1c_7collect12FromIteratorIB3Y_B3l_B4K_EE9from_iterBQ_E0B5v_EB1R_
-__RNvXs2J_NtNtCs8Mbv00yxnRz_4core5slice4iterINtB6_4IterNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectENtNtNtNtBa_4iter6traits8iterator8Iterator4nextBT_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter10store_attr
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_dirINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19str_apply_start_end
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCNvMNtBa_9lazy_lockINtB1a_8LazyLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtCsblfCQaHOzq5_8indexmap3map8IndexMapyNtNtCs2D882usVKTN_10weavepy_vm8gc_trace7SuspectEEE5force0E0B3R_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15leaf_store_attr
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable5drift4sortNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNCINvMNtCshxvaOLs88l5_5alloc5sliceSBW_7sort_byNCNvMs2_B10_NtB10_11Interpreter13sort_with_keys3_0E0EB10_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins8dict_getINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real1b
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject10subclasses
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins17build_super_proxy
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17user_getattribute
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapjTINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync4CellyEEjENtNtB3a_8fasthash13FxBuildHasherEEuE16get_or_init_slowNvNvNtB3a_6object12DICT_WATCHES27___rust_std_internal_init_fnEB3a_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter29store_attr_instance_with_name
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real13construct_ref
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_class
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins9dict_keysINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11dict_valuesINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvCs2D882usVKTN_10weavepy_vm6i64_op
-__RNvMs1_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecyE6resizeCsfGIcTr55AEy_10num_bigint
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins7b_range
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys12sys_getframe
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RINvXs2J_NtNtCs8Mbv00yxnRz_4core5slice4iterINtB7_4IterINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEENtNtNtNtBb_4iter6traits8iterator8Iterator8find_mapNtNtBV_6string6StringNCNvMs2_B1s_NtB1s_11Interpreter24finalize_class_namespaces4_0EB1s_
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20release_finished_gen
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace10untrack_id
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys14asyncgen_hooks
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object13AsyncGenAwaitE9drop_slowBK_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15store_fast_step
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19leaf_load_type_attr
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins7set_addINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_classs0_0EBJ_
-__RNvMsi_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyProperty4fget
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19dispatch_compare_op
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter10cmp_method
-__RNvCs2D882usVKTN_10weavepy_vm8set_view
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter10obj_truthy
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19generator_send_lean
-__RNvMs3_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_7JitHint17every_loop_yields
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry5count
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState24note_weakref_finalizable
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14lean_call_fill
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13inline_finish
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18drop_lean_callable
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real13ref_type_call
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18inline_gen_deliver
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter22load_method_ic_resolve
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyNtNtCs2D882usVKTN_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_20with_key_eq_deferredINtNtCs8Mbv00yxnRz_4core6result6ResultINtNtB2u_6option6OptionbENtNtBY_5error12RuntimeErrorENCNvMs2_BY_NtBY_11Interpreter17leaf_builtin_calls2_0E0TB2p_bEEBY_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11delete_attr
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter22specialized_store_attr
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15do_getattr_call
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11dict_lookup
-__RNvNtCs2D882usVKTN_10weavepy_vm6object12module_class
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter23module_spec_file_origin
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18is_attribute_error
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17ensure_os_environ
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real5build
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject17bump_attr_version
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins10dict_items
-__RNvMsB_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_9DictWatch3new
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24alloc_plain_instance_obj
-__RINvMNtCs2D882usVKTN_10weavepy_vm8lazy_arcINtB3_7LazyArcINtNtB5_4sync7GilCellNtNtB5_6object8DictDataEE10initializeNCNvMse_NtB5_5typesNtB1W_10PyInstance12new_deferreds_0EB5_
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize22type_has_attr_override
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject18setattr_is_default
-__RINvMs10_NtNtNtCshxvaOLs88l5_5alloc11collections5btree4nodeINtB7_16BalancingContextyINtNtBd_3vec3VecINtNtBd_4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotEEE25merge_tracking_child_edgeNtNtBd_5alloc6GlobalEB1W_
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapyuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtBG_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB1y_
-__RINvMs_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3fixINtNtB7_4node7NodeRefNtNtBV_6marker3MutyINtNtBb_3vec3VecINtNtBb_4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotEENtB1f_14LeafOrInternalE31fix_node_and_affected_ancestorsNtNtBb_5alloc6GlobalEB28_
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject21validate_python_bases
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object10PyFunctionE9drop_slowBK_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState10handle_for
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm8gc_trace12note_suspect0E0zEB1y_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14set_insert_key
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18deopt_call_generic
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter22specialized_compare_op
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry18strong_clone_count
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins10b_reversedINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCsfGIcTr55AEy_10num_bigint6bigint6BigIntE9drop_slowCs2D882usVKTN_10weavepy_vm
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15weave_frame_mod9frame_arg
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18maybe_donate_tuple
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectB19_EEEB1d_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object11PyGeneratorE9drop_slowBK_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState21reap_dead_finalizable
-__RNvCs2D882usVKTN_10weavepy_vm15make_gen_method
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15agen_init_hooks
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockyE10initializeNCINvB2_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm8gc_trace27reap_dead_acyclic_amortized0E0zEB1y_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins20apply_set_inplace_op
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18do_issubclass_call
-__RNCNvNtCs2D882usVKTN_10weavepy_vm6object21dict_key_is_reentrant0B5_
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyuNtNtBS_8fasthash13FxBuildHasherE12get_index_ofBO_EBS_
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry24is_native_descr_accessor
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14generator_send
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15reentrant_py_eq
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs2D882usVKTN_10weavepy_vm10QuietEntryEBD_
-__RINvNtCs2D882usVKTN_10weavepy_vm5error10type_errorNtNtCshxvaOLs88l5_5alloc6string6StringEB4_
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object15type_name_owned
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19builtin_classmethod
-__RNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface24install_type_dict_extras12type_prepare
-__RNvCs2D882usVKTN_10weavepy_vm20mangle_class_private
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24finalize_class_namespace
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterTINtNtB9_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotEINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtB1i_6object6ObjectEEENtNtNtB2j_3ops4drop4Drop4dropB1i_
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapjNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEuE16get_or_init_slowNvNvNtB2A_8builtins15DOCSTRING_CACHE27___rust_std_internal_init_fnEB2A_
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE12get_index_ofNtBQ_9LeafProbeEBS_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15do_hasattr_call
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace18take_dead_suspects
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter23run_py_exact_with_cells
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_set
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24run_py_exact_nofree_with
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real24install_proxy_forwarding
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16try_lean_call_kw
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtB2p_5slice4iter4IterINtNtB8_4sync3ArcNtNtB17_5types10TypeObjectEENCNvMs2_B17_NtB17_11Interpreter11build_classs4_0EE9from_iterB17_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBV_6string6StringEE10borrow_mutB7_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter22instantiate_plain_lean
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13instance_plan
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real19find_cached_wrapper
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE12get_index_ofNtBQ_13LeafNameProbeEBS_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins7b_rangeINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvMsn_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_5Range12from_bigints
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15weave_frame_mod5check
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState14note_finalized
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15reap_dead_frame
-__RINvMs_NtNtNtCshxvaOLs88l5_5alloc11collections5btree6removeINtNtB7_4node6HandleINtBY_7NodeRefNtNtBY_6marker3MutyINtNtBb_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm8gc_trace13TrackedHandleENtB1v_4LeafENtB1v_2KVE14remove_leaf_kvNCNvMs5_NtNtB7_3map5entryINtB3G_13OccupiedEntryyB1M_E9remove_kv0NtNtBb_5alloc6GlobalEB27_
-__RNvXs13_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_11PyGeneratorNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNCNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB7_7GcState28reap_dead_finalizable_lockeds_0B9_
-__RNvCs2D882usVKTN_10weavepy_vm15make_agen_await
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15leaf_class_attr
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotE9drop_slowBK_
-__RNvCs2D882usVKTN_10weavepy_vm15is_pep604_union
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins8set_copy
-__RINvNtCs2D882usVKTN_10weavepy_vm13builtin_types25make_exception_with_classReEB4_
-__RNvMNtCs2D882usVKTN_10weavepy_vm5errorNtB2_11PyException9type_name
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real16wrapper_referent
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins29class_matches_classinfo_named
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19type_subclass_match
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace18note_dropped_marks
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtCs2D882usVKTN_10weavepy_vm17InlinePlaceholderE10initializeNCINvB1a_11get_or_initNCNvB1I_18inline_placeholder0E0zE0E0B1I_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_5boxed3BoxNtCs2D882usVKTN_10weavepy_vm9InlineActEE8grow_oneB16_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19binary_subscr_basic
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20del_type_attr_direct
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject22qualified_display_name
-__RNCNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB7_10TypeObject22qualified_display_name0B9_
-__RNvXsq_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys25sys_getfilesystemencoding
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter9iter_next
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins10str_encodeINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1A_EEE4callB6_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNCINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvBG_10str_encodeE0EBI_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins9slot_callINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1y_EEE4callB6_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter26maybe_prompt_reap_replaced
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB2o_11Interpreter26maybe_prompt_reap_replaced0E0zE0E0B2o_
-__RNCNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB7_7GcState18collect_generations0_0B9_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecRNtNtCs2D882usVKTN_10weavepy_vm8gc_trace13TrackedHandleE8grow_oneBR_
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject13lookup_cached
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14descriptor_get
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11build_class
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter22call_metaclass_prepare
-__RNvXNtCsblfCQaHOzq5_8indexmap3mapINtB2_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBM_6ObjectNtNtBO_8fasthash13FxBuildHasherENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBO_
-__RNvXs5_NtCshxvaOLs88l5_5alloc5sliceSINtCsblfCQaHOzq5_8indexmap6BucketNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB17_6ObjectEINtB5_16SpecCloneIntoVecBy_NtNtB7_5alloc6GlobalE10clone_intoB19_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20invoke_init_subclass
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter26check_classcell_propagated
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTyuEE14reserve_rehashNCINvNtB8_3map11make_hasheryuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry12notify_clear
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter18reap_dead_subgraphs_0B7_
-__RNvXs1_NtCs2D882usVKTN_10weavepy_vm13vm_singletonsNtB5_12CascadeGuardNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types19native_seed_for_new
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter9do_imports0_0B7_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectbEEEB1d_
-__RNvMsU_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyFunction4slot
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14code_docstring
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtNtB8_11collections5btree3set8IntoIterNtNtB8_6string6StringEINvMs1j_B15_B13_8from_strB3M_EEE9from_iterB17_
-__RNvMsz_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB5_8IntoIterNtNtBb_6string6StringNtNtB7_7set_val9SetValZSTE10dying_nextCs2D882usVKTN_10weavepy_vm
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins14str_startswithINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins23str_match_prefix_suffix
-__RNvCs2D882usVKTN_10weavepy_vm12attr_name_of
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11list_append
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace12note_suspect
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19drain_if_maybe_dead
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object8contains
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15lean_call_shape
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins18ensure_set_element
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object10PyIteratorEBF_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14binary_op_step
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21specialized_binary_op
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18dispatch_binary_op
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvCs2D882usVKTN_10weavepy_vm5tsdbg0E0zE0E0B2k_
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter18dispatch_binary_ops3_0B7_
-__RNvCs2D882usVKTN_10weavepy_vm19dict_view_set_elems
-__RNvCs2D882usVKTN_10weavepy_vm9binary_op
-__RNvXNtCsblfCQaHOzq5_8indexmap3mapINtB2_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyuNtNtBO_8fasthash13FxBuildHasherENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBO_
-__RNvXs5_NtCshxvaOLs88l5_5alloc5sliceSINtCsblfCQaHOzq5_8indexmap6BucketNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyuEINtB5_16SpecCloneIntoVecBy_NtNtB7_5alloc6GlobalE10clone_intoB19_
-__RNvCs2D882usVKTN_10weavepy_vm10union_sets
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtCsblfCQaHOzq5_8indexmap6BucketNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyuEEEB1J_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14do_sorted_call
-__RNvCs2D882usVKTN_10weavepy_vm30sort_key_needs_dunder_lt_depth
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object3cmp
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins15dict_setdefaultINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins15dict_setdefault
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectbEE8grow_oneBR_
-__RNvXsq_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner15try_allocate_inCsl2apZsWCN6y_12weavepy_capi
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache15is_initializing
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types8exc_attr
-__RNvXs1i_NtCs8Mbv00yxnRz_4core3fmtRNtNtCshxvaOLs88l5_5alloc6string6StringNtB6_7Display3fmtCs2D882usVKTN_10weavepy_vm
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19leaf_load_attr_recv
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecTNtNtB6_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters10filter_map9FilterMapINtNtNtCsblfCQaHOzq5_8indexmap3map4iter4IterNtB1p_7DictKeyB1n_ENCNvMs2_B1r_NtB1r_11Interpreter4stepsS_0EE9from_iterB1r_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15compare_op_step
-__RNvCs2D882usVKTN_10weavepy_vm11compare_str
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins21construct_classmethod
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20set_type_attr_direct
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter26metaclass_descriptor_store
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18try_inline_call_kw
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18lean_call_kw_shape
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecTNtNtB8_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3zip3ZipINtNtB6_9into_iter8IntoIterB14_EIB3s_B1p_EEE9from_iterB1t_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11build_super
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter26load_attr_instance_default
-__RNvCs2D882usVKTN_10weavepy_vm21callable_function_str
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter27dynamic_type_call_with_meta
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecTNtNtB6_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters6cloned6ClonedINtNtB2G_6filter6FilterINtNtNtB2K_5slice4iter4IterB11_ENCNvMs2_B1r_NtB1r_11Interpreter27dynamic_type_call_with_metas0_0EEE9from_iterB1r_
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter24finalize_class_namespaces_0B7_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object10PyIteratorEBF_
-__RNvMsf_NtNtCs8Mbv00yxnRz_4core3str4iterINtB5_13SplitInternalcE9next_backCs2D882usVKTN_10weavepy_vm
-__RINvMNtCs8Mbv00yxnRz_4core3stre18trim_start_matchescECs2D882usVKTN_10weavepy_vm
-__RINvXs2J_NtNtCs8Mbv00yxnRz_4core5slice4iterINtB7_4IterINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEENtNtNtNtBb_4iter6traits8iterator8Iterator3allNCNvMs2_B1s_NtB1s_11Interpreter24finalize_class_namespacesa_0EB1s_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins10dict_itemsINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvCs2D882usVKTN_10weavepy_vm16is_generic_alias
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object10PyDictViewE9drop_slowBK_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjTINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync4CellyEEjEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtB1v_8fasthash13FxBuildHasherE0EB1v_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object9DictWatchEEB11_
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters6cloned6ClonedINtNtB2l_3rev3RevINtNtNtB2p_5slice4iter4IterB13_EEEE9from_iterB17_
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object9type_name
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object9DictWatchEEB11_
-__RNvXsD_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_9DictWatchNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync4CellyEE9drop_slowBL_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11b_frozenset
-__RNvCs2D882usVKTN_10weavepy_vm11bind_method
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter8core_new
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry14is_default_new
-__RNvMse_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10PyInstance12new_deferred
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtCsblfCQaHOzq5_8indexmap6BucketNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB1H_6ObjectEEEB1J_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter34generic_setattr_instance_with_name
-__RNvMsG_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_8DictData20track_deferred_owner
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real16supports_weakref
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterINtNtB9_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB1h_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry8register
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotEE8grow_oneB18_
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageANtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyj7_uE16get_or_init_slowNvNvNtB1j_16weakref_registry12WRAPPER_KEYS27___rust_std_internal_init_fnEB1j_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10PyInstanceE9downgradeBK_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueTNtNtCshxvaOLs88l5_5alloc6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1i_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15subscr_get_step
-__RNvYFG_RL0_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtNtCs8Mbv00yxnRz_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
-__RNvNtCs2D882usVKTN_10weavepy_vm12type_surface21class_getitem_builtin
-__RNvCs2D882usVKTN_10weavepy_vm18make_generic_alias
-__RNvCs2D882usVKTN_10weavepy_vm23collect_type_parameters
-__RNvCs2D882usVKTN_10weavepy_vm15is_typevar_like
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11b_bytearrayINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14b_bytearray_kw
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins15bytes_construct
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins9dict_keys
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11dict_values
-__RNvCs2D882usVKTN_10weavepy_vm9bignum_op
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object6as_i64
-__RNvXs2_NtNtCsfGIcTr55AEy_10num_bigint6bigint7convertNtB7_6BigIntINtNtCs8Mbv00yxnRz_4core7convert4FromxE4from
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19coerce_index_bigint
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtNtCsfGIcTr55AEy_10num_bigint6bigint6BigIntEECs2D882usVKTN_10weavepy_vm
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11do_zip_call
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys16build_with_state0B7_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17frame_locals_view
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins8b_objectINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins8b_object
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11call_object
-__RNvYNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types22install_object_dunders14object_setattrINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types22install_object_dunders14object_setattr
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15weave_frame_mod7is_comp
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15weave_frame_mod15is_module_scope
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtBR_13tuple_storage12TupleStorageEE8grow_oneBR_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19set_frame_gen_owner
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState12fin_make_hot
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16invoke_finalizer
-__RNvMs0_NtCs2D882usVKTN_10weavepy_vm5traceNtB5_20TracingSuppressGuard5enter
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectBX_EENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB11_
-__RNvXs1_NtCs2D882usVKTN_10weavepy_vm5traceNtB5_20TracingSuppressGuardNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvCs2D882usVKTN_10weavepy_vm24generator_frame_traverse
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm8gc_trace11fin_no_cold0E0zE0E0B2m_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB1A_7GcState28reap_dead_finalizable_locked0E0zEB1C_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm8gc_trace13TrackedHandleEEEB1t_
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterINtNtB9_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm8gc_trace13TrackedHandleEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB1h_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockyE10initializeNCINvB1a_11get_or_initNCNvNtCs2D882usVKTN_10weavepy_vm8gc_trace27reap_dead_acyclic_amortized0E0zE0E0B2m_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState24reap_dead_acyclic_locked
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object8PyModuleE9drop_slowBK_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object8PyModuleEBF_
-__RNvMsU_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyFunction5attrs
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyNtNtCs2D882usVKTN_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_13key_cmp_scopebNCNvMs2_BY_NtBY_11Interpreter4stepsy_0E0INtNtCs8Mbv00yxnRz_4core6result6ResultbNtNtBY_5error12RuntimeErrorEEBY_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5types12LeafAttrKindEBF_
-__RNvCs2D882usVKTN_10weavepy_vm15class_attr_fill
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins7set_add
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins17set_apply_inplace
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15core_local_attr
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21leaf_fused_local_attr
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins18set_membership_key
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_classs1_0EBJ_
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter15core_local_attr0B7_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24call_descriptor_accessor
-__RNvYFG_RL0_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtNtCs8Mbv00yxnRz_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types20install_type_dunders12type_mro_get
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16rich_compare_obj
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21deep_equal_collection
-__RNvCs2D882usVKTN_10weavepy_vm10compare_op
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins8set_copyINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter15reap_dead_frame0B7_
-__RNvNtCs2D882usVKTN_10weavepy_vm5error19stop_iteration_with
-__RINvNtCs2D882usVKTN_10weavepy_vm13builtin_types14make_exceptionReEB4_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter26fire_caught_stop_iteration
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtB6_9into_iter8IntoIterINtNtB8_4sync3ArcNtNtB17_5types10TypeObjectEENcNtB13_4Type0EE9from_iterB17_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_classs0_0B7_
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm16weakref_registryNtB4_11WeakRefSlot7upgrade
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins20generic_alias_origin
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11inline_bind
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17inline_bind_cells
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11core_return
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14inline_deliver
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20park_suspended_boxed
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15core_gen_resume
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14try_inline_gen
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17inline_gen_finish
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16leaf_load_method
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14core_gen_yield
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real11ref_type_eq
-__RNvMsh_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionReEE3getB7_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvCs2D882usVKTN_10weavepy_vm14fusion_enabled0E0zEB1w_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvCs2D882usVKTN_10weavepy_vm14fusion_enabled0E0zE0E0B2k_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15load_fast_value
-__RNvNtCs2D882usVKTN_10weavepy_vm5error21attribute_error_named
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtBY_3vec3VecIBU_INtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtB1Q_6object6ObjectEEEEE10initializeNCINvB2_11get_or_initNCNvMsT_B2v_NtB2v_10PyFunction14lean_cells_refs0_0E0zEB1Q_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtB1L_3vec3VecIB1H_INtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtB2F_6object6ObjectEEEEE10initializeNCINvB1a_11get_or_initNCNvMsT_B3k_NtB3k_10PyFunction14lean_cells_refs0_0E0zE0E0B2F_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecPNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectE8grow_oneBR_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21do_list_or_tuple_call
-__RNCINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB4_10str_encodeEs_0B6_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNvNtCs2D882usVKTN_10weavepy_vm8builtins10str_encodeE9drop_slowBK_
-__RNCINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB4_9slot_callEs_0B6_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB1A_11Interpreter26maybe_prompt_reap_replaced0E0zEB1A_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState12collect_impl
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons24drain_pending_cext_drops
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState18collect_generation
-__RNCNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB7_7GcState18collect_generations_0B9_
-__RNvCs2D882usVKTN_10weavepy_vm13iter_traverse
-__RNvMs1e_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_10PyIterator12gc_referents
-__RNCNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB7_7GcState18collect_generations1_0B9_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm9hot_gates9env_flags8reap_dbg0E0zEB1A_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm9hot_gates9env_flags8reap_dbg0E0zE0E0B2o_
-__RINvMs_NtNtNtCshxvaOLs88l5_5alloc11collections5btree6removeINtNtB7_4node6HandleINtBY_7NodeRefNtNtBY_6marker3MutyINtNtBb_3vec3VecINtNtBb_4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotEENtB1v_4LeafENtB1v_2KVE14remove_leaf_kvNCNvMs5_NtNtB7_3map5entryINtB45_13OccupiedEntryyB1M_E9remove_kv0NtNtBb_5alloc6GlobalEB2o_
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm16weakref_registryNtB4_11WeakRefSlot5clear
-__RNCNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB7_7GcState18collect_generationsa_0B9_
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace19clear_object_fields
-__RNvMs_NtCsblfCQaHOzq5_8indexmap5innerINtB4_4CoreNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBM_6ObjectE5clearBO_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtCsblfCQaHOzq5_8indexmap6BucketNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB1a_6ObjectEEB1c_
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterTINtNtB9_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotENtNtB1i_6object6ObjectEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB1i_
-__RNvNtCs2D882usVKTN_10weavepy_vm8gc_trace26sweep_weakref_only_targets
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry7targets
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterTyNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB12_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7io_full5build
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io5build
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io19build_iobase_family
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io13iobase_method
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io19install_fileio_ctor0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io13make_buffered
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io13make_buffered0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io23install_delegating_attr
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io18make_memory_stream
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io18make_memory_stream0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io15set_type_module
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io32make_incremental_newline_decoder
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io32make_incremental_newline_decoder0B7_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEE10initializeNCINvB1a_11get_or_initNCNvNtNtB2i_6stdlib2io27unsupported_operation_class0E0zE0E0B2i_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io8IoFamilyEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io8IoFamilyEBH_
-__RNCNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os22struct_seq_type_layout0se_0B9_
-__RNvCs2D882usVKTN_10weavepy_vm9slice_seq
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os9os_fspath
-__RNCINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB4_9str_splitEs_0B6_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_splitINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1y_EEE4callB6_
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtB2n_3str4iter5SplitReENCNvNtB15_8builtins9str_splits0_0EE9from_iterB15_
-__RNvMsf_NtNtCs8Mbv00yxnRz_4core3str4iterINtB5_13SplitInternalReE4nextCs2D882usVKTN_10weavepy_vm
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins8str_joinINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins8str_join
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins18join_exact_strings
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15subscr_set_step
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter12store_subscr
-__RNvCs2D882usVKTN_10weavepy_vm22apply_slice_assignment
-__RNvCs2D882usVKTN_10weavepy_vm17slice_bound_index
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCshxvaOLs88l5_5alloc3vec6splice6SpliceINtNtBG_9into_iter8IntoIterNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB1P_
-__RNvXs1_NtNtCshxvaOLs88l5_5alloc3vec6spliceINtB5_6SpliceINtNtB7_9into_iter8IntoIterNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB1m_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20leaf_instance_dunder
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter25default_getattribute_slow
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types26make_exception_with_object
-__RNvMs1f_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object4repr
-__RNvNtCs2D882usVKTN_10weavepy_vm6object16bytes_repr_inner
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24build_exception_instance
-__RNvCs2D882usVKTN_10weavepy_vm21lookup_exception_init
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24instantiate_raised_class
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19normalize_exception
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter28break_implicit_context_cycle
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14sync_exc_attrs
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtNtCs2D882usVKTN_10weavepy_vm5error11PyExceptionEEB1e_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_6object11PyTracebackEEE10borrow_mutB7_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object11PyTracebackEBF_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_rfindINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_rfind
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins12str_char_len
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19char_offset_to_byte
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19byte_offset_to_char
-__RNvMsj_NtCs2D882usVKTN_10weavepy_vm12shared_valueNtB5_9SharedStr6repeat
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins10str_rstripINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins10str_rstrip
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VeccEINtB2_18SpecFromIterNestedcNtNtNtCs8Mbv00yxnRz_4core3str4iter5CharsE9from_iterCs2D882usVKTN_10weavepy_vm
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins12str_endswithINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins12str_endswith
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter12do_next_call
-__RNvYFG0_RL1_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectRL0_STNtNtCshxvaOLs88l5_5alloc6string6StringBb_EEINtNtCs8Mbv00yxnRz_4core6result6ResultBb_NtNtBf_5error12RuntimeErrorEINtNtNtB1L_3ops8function2FnTRBa_RBX_EE4callBf_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os10os_stat_kw
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os10dir_fd_arg
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os14path_to_string
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtB4_2io5error5ErrorECs2D882usVKTN_10weavepy_vm
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins8str_findINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvXsv_NtNtCs8Mbv00yxnRz_4core3str7patternNtB5_11StrSearcherNtB5_8Searcher10next_match
-__RNCINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB4_15bytes_decode_kwEs_0B6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_lower
-__RNvNtCs2D882usVKTN_10weavepy_vm12unicode_case5lower
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os15stat_seq_finish
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter7repr_of
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3ucd10case_flags
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm8AttrPolyEE10initializeNCINvB2_11get_or_initNCNvB1v_14code_attr_poly0E0zEB1v_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11list_extendINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvMs7_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10TypeObject6lookup
-__RNvCs2D882usVKTN_10weavepy_vm17resolve_metaclass
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21invoke_set_name_hooks
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEEB1t_
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons13enter_cascade
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11import_star
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RNvXs2J_NtNtCs8Mbv00yxnRz_4core5slice4iterINtB6_4IterNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectENtNtNtNtBa_4iter6traits8iterator8Iterator9size_hintBT_
-__RNvMsU_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyFunction8set_slot
-__RNvCs2D882usVKTN_10weavepy_vm15instance_method
-__RNvNtCs2D882usVKTN_10weavepy_vm5error21io_error_to_py_named2
-__RNvXsv_NtNtCs8Mbv00yxnRz_4core3str7patternNtB5_11StrSearcherNtB5_8Searcher10next_match
-__RNvNtCs2D882usVKTN_10weavepy_vm5error24set_oserror_filename_obj
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter29leaf_resolve_instance_attr_ix
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs2D882usVKTN_10weavepy_vm8LeafAttrEBD_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_5types10TypeObjectEE6borrowB7_
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter21specialized_load_attr0B7_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins8str_find
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins15bytes_decode_kwINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1F_EEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins10bytes_data
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNvNtCs2D882usVKTN_10weavepy_vm8builtins15bytes_decode_kwE9drop_slowBK_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os21stat_result_from_meta
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm8AttrPolyEE10initializeNCINvB1a_11get_or_initNCNvB2i_14code_attr_poly0E0zE0E0B2i_
-__RNvYNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types22install_object_dunders11object_initINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types22install_object_dunders11object_init
-__RNvMsd_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_11SlotStorage8index_of
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19run_py_exact_nofree
-__RNvMse_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_10PyInstance11try_recycle
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10PyInstanceEE8grow_oneB17_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14load_attr_type
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry15is_surface_only
-__RNvXsF_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_8DictDataNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14core_pure_call
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_lowerINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11list_extend
-__RNvMs_NtCsblfCQaHOzq5_8indexmap5innerINtB4_4CoreyNtNtCs2D882usVKTN_10weavepy_vm8gc_trace7SuspectE18swap_remove_finishBP_
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18is_data_descriptor
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os10os_listdir
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os9os_getgid
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter12format_value
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21core_pure_global_call
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2os9os_getuid
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod31register_current_thread_pthread
-__RNvXsq_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins8dict_popINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12warnings_mod15default_filters
-__RNvMs3_NtCs2EXTltNQfP1_17cranelift_codegen3isaDNtB5_9TargetIsaEL_15frontend_config
-__RNvMs2_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir8functionNtB5_8Function19with_name_signature
-__RNvMNtCs2EXTltNQfP1_17cranelift_codegen7contextNtB2_7Context12for_function
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2EXTltNQfP1_17cranelift_codegen3isa10IsaBuilderINtNtB4_6result6ResultINtNtCshxvaOLs88l5_5alloc4sync3ArcDNtBE_9TargetIsaEL_ENtNtBG_6result12CodegenErrorEEECseWU5rFlE7Md_11weavepy_jit
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs8Mbv00yxnRz_4core4cell4CellTyyEEE4withNCNvMNtNtBa_4hash6randomNtB1I_11RandomState3new0B21_ECseWU5rFlE7Md_11weavepy_jit
-__RNvMsi_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB5_8BTreeMapjNtNtB7_7set_val9SetValZSTE6insertCseWU5rFlE7Md_11weavepy_jit
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner15try_allocate_inCseWU5rFlE7Md_11weavepy_jit
-__RNvMs3_NtNtCshxvaOLs88l5_5alloc11collections9vec_dequeINtB5_8VecDequejE13push_back_mutCseWU5rFlE7Md_11weavepy_jit
-__RINvXNtNtCshxvaOLs88l5_5alloc3vec14spec_from_elemINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeENtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEB1r_
-__RNCNCNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB9_8JitState12get_compileds0_0s1_0Bb_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze11infer_block
-__RINvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB6_7HashMapjmNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE12contains_keyjECseWU5rFlE7Md_11weavepy_jit
-__RNCNCNvNtCs2D882usVKTN_10weavepy_vm5tier212warm_compile0s2_0B7_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze10emit_block
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins14str_replace_kwINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1E_EEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize26attempt_specialize_call_kw
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real26instance_method_trampoline
-__RNvYFG_RL0_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtNtCs8Mbv00yxnRz_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
-__RNvNtCs2EXTltNQfP1_17cranelift_codegen3isa6lookup
-__RNvXs1_NtCs2EXTltNQfP1_17cranelift_codegen3isaINtB5_10IsaBuilderINtNtCs8Mbv00yxnRz_4core6result6ResultINtNtCshxvaOLs88l5_5alloc4sync3ArcDNtB5_9TargetIsaEL_ENtNtB7_6result12CodegenErrorEENtNtB7_8settings12Configurable6enableCsgdF30VMnbvd_16cranelift_native
-__RNvXs_NtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch64NtB4_14AArch64BackendNtB6_9TargetIsa5flags
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215probe_attr_lane
-__RNvCsgdF30VMnbvd_16cranelift_native7builder
-__RNvCsgdF30VMnbvd_16cranelift_native18infer_native_flags
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit7analyze2SEE8grow_oneBQ_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7imp_mod5build
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7imp_mod7builtin
-__RNvMsi_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyProperty6reinit
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins22construct_staticmethod
-__RNvMCs2D882usVKTN_10weavepy_vmNtB2_5Frame3top
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize34attempt_specialize_unpack_sequence
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18do_isinstance_call
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21deopt_load_attr_taken
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11run_lean_fn
-__RNvYFG_RL0_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtNtCs8Mbv00yxnRz_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter16module_shadowings0_0B7_
-__RNCINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB4_14str_replace_kwEs_0B6_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7imp_mod14imp_is_builtin
-__RNvNtCs2D882usVKTN_10weavepy_vm10specialize26kw_uncovered_have_defaults
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBF_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real1b
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real26instance_dunder_trampoline
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real14module_getattr
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11dict_remove
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19kw_names_bind_check
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs8Mbv00yxnRz_4core4cell7RefCellNtNtCs2D882usVKTN_10weavepy_vm5tier28JitStateEE4withNCNvB1x_12warm_compile0uEB1z_
-__RNvXs1_NtCs2EXTltNQfP1_17cranelift_codegen8settingsNtB5_7BuilderNtB5_12Configurable3set
-__RNvXs1_NtCs2EXTltNQfP1_17cranelift_codegen8settingsNtB5_7BuilderNtB5_12Configurable6enable
-__RNvXs_NtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch64NtB4_14AArch64BackendNtB6_9TargetIsa6triple
-__RNvMNtCs2EXTltNQfP1_17cranelift_codegen7contextNtB2_7Context3new
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze12analyze_impl
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze12analyze_once
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze12plan_rewrite
-__RNvXNtCs3fRkGY2FHAU_9hashbrown3mapINtB2_7HashMapjuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCseWU5rFlE7Md_11weavepy_jit
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze12build_blocks
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjjNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCseWU5rFlE7Md_11weavepy_jit
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRjECseWU5rFlE7Md_11weavepy_jit
-__RINvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB6_7HashMapjuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE12contains_keyjECseWU5rFlE7Md_11weavepy_jit
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze16reachable_blocks
-__RNvMs3_NtNtCshxvaOLs88l5_5alloc11collections9vec_dequeINtB5_8VecDequejE4growCseWU5rFlE7Md_11weavepy_jit
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRmECseWU5rFlE7Md_11weavepy_jit
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCseWU5rFlE7Md_11weavepy_jit
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit7analyze5ESlotE8grow_oneBQ_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze14emit_attr_lane
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecmEINtB2_18SpecFromIterNestedmINtNtNtNtCs82bWklYMk3w_3std11collections4hash3set8IntoItermEE9from_iterCseWU5rFlE7Md_11weavepy_jit
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir13RangeLoopMetaEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtB2u_5slice4iter4IterB13_ENCNvNtB17_7analyze12analyze_onces3_0EE9from_iterB17_
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12ListLoopMetaEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtB2n_6filter6FilterINtNtNtB2r_5slice4iter4IterB11_ENCNvNtB15_7analyze12analyze_onces4_0ENCB41_s5_0EE9from_iterB15_
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecmEINtB2_18SpecFromIterNestedmINtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtNtCs82bWklYMk3w_3std11collections4hash3set4IterjENCNvNtCseWU5rFlE7Md_11weavepy_jit7analyze12analyze_oncesb_0EE9from_iterB3i_
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecmEINtB2_18SpecFromIterNestedmINtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtNtCs82bWklYMk3w_3std11collections4hash3map6ValuesjNtNtCseWU5rFlE7Md_11weavepy_jit7analyze14CompHeaderPlanENCNvB3f_12analyze_oncesc_0EE9from_iterB3h_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB4_6option6OptionIBC_NtNtCseWU5rFlE7Md_11weavepy_jit7analyze2SEEEEEB1C_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins23reentrant_load_attr_opt
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins15property_setterINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins16property_payload
-__RNvMsi_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyProperty4fdel
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14wstr_attr_dict
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18ensure_module_spec
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter16bind_python_argss8_0B7_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter27specialized_unpack_sequence
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter25recursive_isinstance_type
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7imp_mod13imp_is_frozen
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write9write_str
-__RINvMs3_NtCshxvaOLs88l5_5alloc3stre7replaceReECs2D882usVKTN_10weavepy_vm
-__RNCNvMNtCs2D882usVKTN_10weavepy_vm13builtin_typesNtB4_12BuiltinTypes5builds4_0B6_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7imp_mod18imp_create_builtin
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real5build
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexINtNtNtNtB7_11collections4hash3map7HashMapyjEEE10initializeNCINvB2_11get_or_initNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod16pthread_registry0E0zEB2M_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTyjEE14reserve_rehashNCINvNtB8_3map11make_hasheryjNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins8dict_pop
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20kw_names_fill_locals
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12warnings_mod5build
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12warnings_mod10builtin_kw
-__RNvYFG0_RL1_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectRL0_STNtNtCshxvaOLs88l5_5alloc6string6StringBb_EEINtNtCs8Mbv00yxnRz_4core6result6ResultBb_NtNtBf_5error12RuntimeErrorEINtNtNtB1L_3ops8function2FnTRBa_RBX_EE4callBf_
-__RNvYNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12warnings_mod5builds_0INtNtNtCs8Mbv00yxnRz_4core3ops8function6FnOnceTRSNtNtBa_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1R_EEE9call_onceBa_
-__RNvXsq_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RINvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB6_7HashMapPNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectNtNtCs2D882usVKTN_10weavepy_vm5tier210CacheEntryNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE7get_mutBO_EB1D_
-__RNvMs0_NtCseWU5rFlE7Md_11weavepy_jit6engineNtB5_9JitEngine3new
-__RNvMs0_NtCs2EXTltNQfP1_17cranelift_codegen8settingsNtB5_7Builder6lookup
-__RNvMsn_NtCs2EXTltNQfP1_17cranelift_codegen8settingsNtB5_5Flags3new
-__RNvYNCNvNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch6411isa_builder0INtNtNtCs8Mbv00yxnRz_4core3ops8function6FnOnceTNtNtCshfBCtCVroC4_14target_lexicon6triple6TripleNtNtBa_8settings5FlagsRNtB2G_7BuilderEE9call_onceBa_
-__RNvMNtCslsAwG9FmMO9_13cranelift_jit7backendNtB2_10JITBuilder8with_isa
-__RNvMs0_NtCslsAwG9FmMO9_13cranelift_jit7backendNtB5_9JITModule3new
-__RNvXs_NtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch64NtB4_14AArch64BackendNtB6_9TargetIsa20page_size_align_log2
-__RNvMs3_NtCs2EXTltNQfP1_17cranelift_codegen3isaDNtB5_9TargetIsaEL_17default_call_conv
-__RNvMNtCsgdPmlPE1JLX_18cranelift_frontend8frontendNtB2_22FunctionBuilderContext3new
-__RNCNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB7_8JitState12get_compileds0_0B9_
-__RNvXs3_NtNtCs8Mbv00yxnRz_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCseWU5rFlE7Md_11weavepy_jit
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze14compute_livein
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmuEE14reserve_rehashNCINvNtB8_3map11make_hashermuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECsgdPmlPE1JLX_18cranelift_frontend
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapmuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCseWU5rFlE7Md_11weavepy_jit
-__RNvXNtCs3fRkGY2FHAU_9hashbrown3mapINtB2_7HashMapmuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneCseWU5rFlE7Md_11weavepy_jit
-__RNvXsE_NtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB5_9CtorStateNtNtCs8Mbv00yxnRz_4core7default7Default7default
-__RINvXNtNtCshxvaOLs88l5_5alloc3vec14spec_from_elemINtNtCs8Mbv00yxnRz_4core6option6OptionINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit7analyze2SEEENtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEB1B_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit7analyze2SEENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze13step_abstract
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCseWU5rFlE7Md_11weavepy_jit
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15infer_load_attr
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB2_9PathArena5names
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214walk_attr_path
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier220attr_fingerprint_obj
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECseWU5rFlE7Md_11weavepy_jit
-__RINvXNtNtCshxvaOLs88l5_5alloc3vec14spec_from_elemINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit2ir6TBlockENtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEB1r_
-__RNvXNtNtNtCshxvaOLs88l5_5alloc11collections9vec_deque14spec_from_iterINtB4_8VecDequejEINtB2_12SpecFromIterjINtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters6copied6CopiedINtNtNtB1Q_5slice4iter4IterjEEE14spec_from_iterCseWU5rFlE7Md_11weavepy_jit
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze10emit_instr
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit2ir5TStmtE8grow_oneBQ_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12AttrSiteMetaE8grow_oneBQ_
-__RNvMs3_NtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB5_5ESlot8is_plain
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12IterLoopMetaEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtB2n_6filter6FilterINtNtNtB2r_5slice4iter4IterNtB13_12ListLoopMetaENCNvNtB15_7analyze12analyze_onces6_0ENCB4h_s7_0EE9from_iterB15_
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir13CompSavedMetaEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters6copied6CopiedINtNtNtNtCs82bWklYMk3w_3std11collections4hash3map6ValuesmB11_EEE9from_iterB15_
-__RINvMs_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecmE8dedup_byNCNvMs4_B5_Bv_5dedup0ECseWU5rFlE7Md_11weavepy_jit
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit7analyze9CtorStateEBF_
-__RNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree8contains
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins18construct_property
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins21property_init_members
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins13property_with
-__RNvMsi_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyProperty4fset
-__RNvMsi_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_10PyProperty3doc
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19do_dict_update_call
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15dict_merge_from
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit7analyze8RawBlockEEB1c_
-__RNvMs0_NtCseWU5rFlE7Md_11weavepy_jit6engineNtB5_9JitEngine13compile_tfunc
-__RNvYNtNtCslsAwG9FmMO9_13cranelift_jit7backend9JITModuleNtNtCs8MhVFxlpMWQ_16cranelift_module6module6Module13clear_contextCseWU5rFlE7Md_11weavepy_jit
-__RNvMs3_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph5clear
-__RNvMs7_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir8constantNtB5_12ConstantPool5clear
-__RNvMs2_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir8functionNtB5_8Function5clear
-__RNvMsz_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB5_8IntoIterNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities4InstINtCsfNVduRRWYaN_8smallvec8SmallVecANtNtB1b_15user_stack_maps17UserStackMapEntryj4_EE10dying_nextB1d_
-__RNvMsz_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB5_8IntoIterNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8constant12ConstantDataNtNtB1b_8entities8ConstantE10dying_nextB1d_
-__RINvNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECsgdPmlPE1JLX_18cranelift_frontend
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsgdPmlPE1JLX_18cranelift_frontend
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7extfunc8AbiParamE8grow_oneBS_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfg15ValueDataPackedE8grow_oneBS_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCs2EXTltNQfP1_17cranelift_codegen8machinst16CompiledCodeBaseNtNtB10_6buffer5FinalEEEB12_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs2EXTltNQfP1_17cranelift_codegen
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfg9BlockDataE8grow_oneBS_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockNtNtCsgdPmlPE1JLX_18cranelift_frontend3ssa12SSABlockDataE20resize_for_index_mutB1X_
-__RNvMs3_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_10EntityListNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE4pushB16_
-__RINvNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs2EXTltNQfP1_17cranelift_codegen
-__RNvXs0_NtCsgdPmlPE1JLX_18cranelift_frontend8frontendNtB5_15FuncInstBuilderNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7builder15InstBuilderBase5build
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockNtNtB12_6layout9BlockNodeE20resize_for_index_mutB14_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockNtNtCsgdPmlPE1JLX_18cranelift_frontend8frontend11BlockStatusE20resize_for_index_mutB1X_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructions15InstructionDataE8grow_oneBS_
-__RNvMsb_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph17make_inst_results
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir6layoutNtB5_6Layout11append_inst
-__RNvMs3_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_10EntityListNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE8as_sliceB16_
-__RNvMsb_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph35non_tail_call_or_try_call_signature
-__RNvMsc_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB5_15InstructionData12analyze_call
-__RNvXNvMsb_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB8_13DataFlowGraph17inst_result_typesNtB2_15InstResultTypesNtNtNtNtCs8Mbv00yxnRz_4core4iter6traits8iterator8Iterator4next
-__RNvMsf_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB5_17OperandConstraint7resolve
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities4InstNtNtB12_6layout8InstNodeE20resize_for_index_mutB14_
-__RNvMs0_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir6layoutNtB5_6Layout15assign_inst_seq
-__RNvXNtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB2_5InstsINtNtNtCs8Mbv00yxnRz_4core3ops5index5IndexNtNtB4_8entities4InstE5index
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir5types4TypeE8grow_oneBS_
-__RNvMs4_NtCsgdPmlPE1JLX_18cranelift_frontend8frontendNtB5_15FunctionBuilder11try_def_var
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtCsgdPmlPE1JLX_18cranelift_frontend8variable8VariableIBF_NtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockINtNtB4_13packed_option12PackedOptionNtB1Y_5ValueEEE20resize_for_index_mutB12_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockINtNtB4_13packed_option12PackedOptionNtB10_5ValueEE20resize_for_index_mutCsgdPmlPE1JLX_18cranelift_frontend
-__RNvMs3_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_10EntityListNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE4pushCseWU5rFlE7Md_11weavepy_jit
-__RNvMs3_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_10EntityListNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE4growCseWU5rFlE7Md_11weavepy_jit
-__RNvMNtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB2_9BlockCall5block
-__RNvMs1_NtCsgdPmlPE1JLX_18cranelift_frontend3ssaNtB5_10SSABuilder25declare_block_predecessor
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser16parse_dict_value
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser17expect_dict_colon
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEE8grow_oneB1s_
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler15compile_subdict
-__RNvXsZ_NtCs1LCFj3BjrAK_14weavepy_parser3astNtB5_8ExprKindNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvCs6MP6LIxl4z2_16weavepy_compiler25expr_contains_typeparamfn
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEEB1a_
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser11parse_yield
-__RNvCs6MP6LIxl4z2_16weavepy_compiler31collect_inlined_comp_inner_free
-__RNvMs10_NtNtNtCshxvaOLs88l5_5alloc11collections5btree4nodeINtB6_16BalancingContextyINtNtBc_3vec3VecINtNtBc_4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotEEE15bulk_steal_leftB1V_
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler23compile_slice_two_parts
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapymNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCs6MP6LIxl4z2_16weavepy_compiler
-__RINvNvNtCs6MP6LIxl4z2_16weavepy_compiler6intern13constant_hash4hashNtNtNtCs82bWklYMk3w_3std4hash6random13DefaultHasherEB6_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTymEE14reserve_rehashNCINvNtB8_3map11make_hasherymNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtCs8Mbv00yxnRz_4core6option6OptionmEE8grow_oneCs6MP6LIxl4z2_16weavepy_compiler
-__RNvCs6MP6LIxl4z2_16weavepy_compiler18collect_comp_reads
-__RINvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB6_7HashMapNtNtCshxvaOLs88l5_5alloc6string6StringuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6removeBO_ECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast13ComprehensionEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast13ComprehensionECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler29collect_global_walrus_shallow
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringNtCs6MP6LIxl4z2_16weavepy_compiler7CompSymEE8grow_oneB1a_
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler21collect_names_shallow
-__RNvCs6MP6LIxl4z2_16weavepy_compiler12comp_inlines
-__RNvCs6MP6LIxl4z2_16weavepy_compiler20expr_yields_in_scope
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler14comp_own_cells
-__RNvCs6MP6LIxl4z2_16weavepy_compiler29collect_comp_scope_inner_free
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler12comp_symbols
-__RNvCs6MP6LIxl4z2_16weavepy_compiler28collect_target_names_ordered
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler27collect_nested_comp_symbols
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringINtNtCs8Mbv00yxnRz_4core6option6OptionNtCs6MP6LIxl4z2_16weavepy_compiler7BindingEEE8grow_oneB1M_
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapmNtNtCs6MP6LIxl4z2_16weavepy_compiler8bytecode6OpCodeNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertBS_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmNtNtCs6MP6LIxl4z2_16weavepy_compiler8bytecode6OpCodeEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBV_
-__RNvMNtCsftQRLRmRWkP_13weavepy_lexer7scannerNtB2_7Scanner19scan_fstring_extent
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVeccE8grow_oneCsftQRLRmRWkP_13weavepy_lexer
-__RNvMNtCsftQRLRmRWkP_13weavepy_lexer7scannerNtB2_7Scanner25scan_fstring_field_extent
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser17fstring_parts_for
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser24parse_fstring_body_inner
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser19parse_fstring_field
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser28scrub_fstring_field_comments
-__RINvMNtCs8Mbv00yxnRz_4core3stre12trim_matchesNCNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtBM_6Parser19parse_fstring_field0EBO_
-__RNvXs1i_NtCs8Mbv00yxnRz_4core3fmtReNtB6_7Display3fmtCs1LCFj3BjrAK_14weavepy_parser
-__RNvNtCsftQRLRmRWkP_13weavepy_lexer7scanner8tokenize
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser16remap_expr_spans
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser17parse_import_from
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast7KeywordECs6MP6LIxl4z2_16weavepy_compiler
-__RINvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_expr_inners3_0EB6_
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler7ast_opt10eval_binop
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler7ast_opt6int_of
-__RNvXs2_NtNtCsfGIcTr55AEy_10num_bigint6bigint7convertNtB7_6BigIntINtNtCs8Mbv00yxnRz_4core7convert4FromxE4from
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler7ast_opt10str_points
-__RNvMNtCshxvaOLs88l5_5alloc5sliceSm6repeatCs6MP6LIxl4z2_16weavepy_compiler
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler7ast_opt8make_str
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler7ast_opt15optimize_format
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VeccEINtB2_18SpecFromIterNestedcNtNtNtCs8Mbv00yxnRz_4core3str4iter5CharsE9from_iterCs6MP6LIxl4z2_16weavepy_compiler
-__RNvXsZ_NtCs1LCFj3BjrAK_14weavepy_parser3astNtB5_8ExprKindNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvXs1b_NtCs1LCFj3BjrAK_14weavepy_parser3astNtB6_8ConstantNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEECs6MP6LIxl4z2_16weavepy_compiler
-__RNvCs6MP6LIxl4z2_16weavepy_compiler14constant_slice
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEBD_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueTNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantBC_BC_EEBE_
-__RNvXs2_NtNtCs8Mbv00yxnRz_4core5slice3cmpNtNtCshxvaOLs88l5_5alloc6string6StringINtB5_14SlicePartialEqBC_E17equal_same_lengthCs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftNtNtCshxvaOLs88l5_5alloc6string6StringNCINvMB8_SB1m_20sort_unstable_by_keyTjjB1m_ENCNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB2O_8Compiler6finish0E0EB2O_
-__RNCINvMNtCs8Mbv00yxnRz_4core5sliceSNtNtCshxvaOLs88l5_5alloc6string6String20sort_unstable_by_keyTjjBx_ENCNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB1K_8Compiler6finish0E0B1K_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftTjjENvYB1m_NtNtBa_3cmp10PartialOrd2ltECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser11parse_raise
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser16parse_else_block
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler15emit_super_attr
-__RNvNtCs6MP6LIxl4z2_16weavepy_compiler6intern9name_hash
-__RNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree18write_install_json
-__RNCNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree18write_install_json0B5_
-__RINvXNtNtCs4w2dA8fTJLO_10serde_json5value3serNtB5_5ValueNtNtCs2GFHAg14Vqm_10serde_core3ser9Serialize9serializeQINtNtB7_3ser10SerializerQINtNtCshxvaOLs88l5_5alloc3vec3VechENtB1O_15PrettyFormatterEECs2D882usVKTN_10weavepy_vm
-__RNvXs1_NtCs4w2dA8fTJLO_10serde_json3serQINtB5_10SerializerQINtNtCshxvaOLs88l5_5alloc3vec3VechENtB5_15PrettyFormatterENtNtCs2GFHAg14Vqm_10serde_core3ser10Serializer13serialize_strCs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree12write_atomic
-__RINvNvXs0_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB8_8BTreeMappppENtNtCs8Mbv00yxnRz_4core5clone5Clone5clone13clone_subtreeNtNtBe_6string6StringNtNtCs4w2dA8fTJLO_10serde_json5value5ValueNtNtBe_5alloc6GlobalECs2D882usVKTN_10weavepy_vm
-__RNvXs_NtNtCs4w2dA8fTJLO_10serde_json5value5indexeNtB4_5Index15index_or_insert
-__RNvNvNtCs2D882usVKTN_10weavepy_vm11stdlib_tree12uname_fields5field
-__RINvMs3_NtCshxvaOLs88l5_5alloc3stre7replacecECs2D882usVKTN_10weavepy_vm
-__RNvYNCNvMNtCsftQRLRmRWkP_13weavepy_lexer7scannerNtB7_7Scanner11scan_number0INtNtNtCs8Mbv00yxnRz_4core3ops8function6FnOnceThEE9call_onceB9_
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser9parse_del
-__RNvMse_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8Compiler14compile_delete
-__RNvMs10_NtNtNtCshxvaOLs88l5_5alloc11collections5btree4nodeINtB6_16BalancingContextyINtNtBc_3vec3VecINtNtBc_4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm16weakref_registry11WeakRefSlotEEE16bulk_steal_rightB1V_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14format_obj_str
-__RNvCs2D882usVKTN_10weavepy_vm20format_via_spec_impl
-__RNvXsv_NtNtCs8Mbv00yxnRz_4core3str7patternNtB5_11StrSearcherNtB5_8Searcher10next_match
-__RINvNtCshxvaOLs88l5_5alloc3str17join_generic_copyehReECs2D882usVKTN_10weavepy_vm
-__RNvXsq_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RNvXsP_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8ConstantNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter30register_source_with_linecache
-__RNCINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB4_14str_splitlinesEs_0B6_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins14str_splitlinesINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1E_EEE4callB6_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNvNtCs2D882usVKTN_10weavepy_vm8builtins14str_splitlinesE9drop_slowBK_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins8list_popINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins8list_pop
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19code_synthetic_attr
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtB2l_6cloned6ClonedINtNtNtB2p_5slice4iter4IterNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEENvB17_25constant_to_object_publicEE9from_iterB17_
-__RNvXsP_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_8ConstantNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectE9drop_slowBI_
-__RNvXs9_NtCs6MP6LIxl4z2_16weavepy_compiler8bytecodeNtB5_10CacheTableNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs6MP6LIxl4z2_16weavepy_compiler5VmExtEBD_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEEB1a_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtCs6MP6LIxl4z2_16weavepy_compiler13WireOverridesEEEB1y_
-__RNvXsr_Cs6MP6LIxl4z2_16weavepy_compilerNtB5_10CodeObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24run_interpreter_shutdown
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10atexit_mod17run_exit_handlers
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib16interpreters_mod8RegistryEE10initializeNCINvB2_11get_or_initNCNvB1o_8registry0E0zEB1s_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtNtBa_6poison5mutex5MutexNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib16interpreters_mod8RegistryEE10initializeNCINvB1a_11get_or_initNCNvB2b_8registry0E0zE0E0B2f_
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecyEINtB2_18SpecFromIterNestedyINtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters6copied6CopiedINtNtNtNtCs82bWklYMk3w_3std11collections4hash3map4KeysyNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib16interpreters_mod16InterpreterEntryEEE9from_iterB3n_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtB1a_5error12RuntimeErrorEEEB1a_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB1a_6object6ObjectEEEB1a_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyANtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyj7_EB19_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtB1a_6option6OptionINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtNtNtCs2D882usVKTN_10weavepy_vm5types10type_cache5EntryEEEEB2J_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter23run_shutdown_finalizers
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13flush_streams
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16current_sys_attr
-__RNvMs19_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6PyFile9is_closed
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtBV_6string6StringNtNtB7_6object6ObjectEEE6borrowB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins15file_check_open
-__RNvMs19_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6PyFile15flush_write_buf
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellDNtNtCs82bWklYMk3w_3std2io5WriteNtNtCs8Mbv00yxnRz_4core6marker4SyncNtB1o_4SendEL_E10borrow_mutB7_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs82bWklYMk3w_3std4path7PathBufEECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs2D882usVKTN_10weavepy_vm16InterpreterFlagsECsdp43gqXvMwI_11weavepy_cli
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCsdp43gqXvMwI_11weavepy_cli12EnvOverridesEBD_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCsdp43gqXvMwI_11weavepy_cli3CliEBD_
-__RNvNtCsdp43gqXvMwI_11weavepy_cli6pcprof6finish
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtB1a_6object6ObjectB2l_EEEEB1a_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectB19_EEEB1d_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm3gil8GilGuardEEEB2h_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyNtNtCs2D882usVKTN_10weavepy_vm6object18KeyComparisonStateEB19_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB19_6option6OptionTjNtNtCs2D882usVKTN_10weavepy_vm12shared_value9SharedStrjEEEEEB2F_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellNtNtCs2D882usVKTN_10weavepy_vm5tier28JitStateEEB1J_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs2D882usVKTN_10weavepy_vm16CodeConstObjectsEBD_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtNtNtBa_11collections4hash3set7HashSetNtNtB19_12shared_value9SharedStrEEEB19_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtCs6MP6LIxl4z2_16weavepy_compiler13WireOverridesEEECs2D882usVKTN_10weavepy_vm
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB1a_13vm_singletons13ThreadHandlesEEEB1a_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib16faulthandler_mod16RegisteredThreadEBH_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecONtB1a_11InterpreterEEEB1a_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtNtNtBa_11collections4hash3map7HashMapReTINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB19_5types10TypeObjectERNtNtNtB19_6stdlib2os15StructSeqLayoutEEEEB19_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapNtNtCshxvaOLs88l5_5alloc6string6StringINtNtB19_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEEB3o_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtB19_13tuple_storage12TupleStorageEEB19_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtBG_13tuple_storage12TupleStorageEEBG_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyNtNtCs2D882usVKTN_10weavepy_vm6tcache9ExitGuardEB19_
-__RNvXsa_Cs2D882usVKTN_10weavepy_vmNtB5_9LeanFrameNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvMs19_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6PyFile14get_extra_attr
-__RNvMs19_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6PyFile5flush
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecTNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB16_6ObjectEEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtB6_5drain5DrainINtCsblfCQaHOzq5_8indexmap6BucketB14_B1N_EENvMs0_B3H_B3E_9key_valueEE9from_iterB18_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB1b_6ObjectEEEB1d_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtCs6MP6LIxl4z2_16weavepy_compiler8ConstantEEB1a_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs6MP6LIxl4z2_16weavepy_compiler5VmExtECs2D882usVKTN_10weavepy_vm
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtB1a_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm13vm_singletons23FaulthandlerThreadGuardEEEB27_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm8gc_traceNtB5_7GcState23finalization_candidates
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm8gc_trace13TrackedHandleEEEB1t_
-__RINvMs_NtCsblfCQaHOzq5_8indexmap5innerINtB5_4CoreNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBN_6ObjectE5drainNtNtNtCs8Mbv00yxnRz_4core3ops5range9RangeFullEBP_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins10file_flushINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins10file_flush
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcDNtNtCs8Mbv00yxnRz_4core3any3AnyNtNtBL_6marker4SyncNtB1e_4SendEL_E9drop_slowCs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs82bWklYMk3w_3std4sync9once_lock8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm10MethodSlotEEEB20_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs82bWklYMk3w_3std4sync9once_lock8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm9StampSlotEEEB20_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs82bWklYMk3w_3std4sync9once_lock8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm8AttrPolyEEEB20_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs82bWklYMk3w_3std4sync9once_lock8OnceLockINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtCs2D882usVKTN_10weavepy_vm8CallSlotEEEB20_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc4sync4WeakDNtNtB4_3any3AnyNtNtB4_6marker4SyncNtB1s_4SendEL_RNtNtBG_5alloc6GlobalEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtNtNtBa_11collections4hash3map7HashMapReNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectEEEB19_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib16faulthandler_mod16note_thread_exit
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapTjNtNtCshxvaOLs88l5_5alloc6string6StringENtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB34_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtB1a_6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm13builtin_types12BuiltinTypesEEEEB2F_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtNtCs82bWklYMk3w_3std6thread9lifecycle6PacketlEE9drop_slowCsdp43gqXvMwI_11weavepy_cli
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtB1a_6object14ReprGuardStateEEB1a_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtCshxvaOLs88l5_5alloc3vec3VecTjNtNtCs2D882usVKTN_10weavepy_vm8builtins12SetInplaceOpEEEEB2j_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectB1H_EEEB1L_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB1L_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10PyInstanceEEEEB2z_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapjTINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync4CellyEEjENtNtB30_8fasthash13FxBuildHasherEEEB30_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB1a_5types10TypeObjectEEEEB1a_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapjNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB2q_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellNtNtB19_5trace15MonitoringToolsEEB19_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtNtNtBa_11collections4hash3set7HashSetNtNtCshxvaOLs88l5_5alloc6string6StringEEEB19_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapNtNtBa_4path7PathBufTNtNtCs2D882usVKTN_10weavepy_vm6import8DirStampINtNtCshxvaOLs88l5_5alloc2rc2RcINtNtB1K_3set7HashSetNtNtNtBa_3ffi6os_str8OsStringEEEEEEB2K_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyEBG_
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcINtNtNtNtCs82bWklYMk3w_3std11collections4hash3set7HashSetNtNtNtBM_3ffi6os_str8OsStringEE9drop_slowCs2D882usVKTN_10weavepy_vm
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer5guard
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer9emit_exit
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7extfunc9SignatureE8grow_oneBS_
-__RNvMsc_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB5_15InstructionData15mask_immediates
-__RNvMs2_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_8ListPoolNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE7reallocCseWU5rFlE7Md_11weavepy_jit
-__RINvNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECseWU5rFlE7Md_11weavepy_jit
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer10emit_block
-__RNvMs4_NtCsgdPmlPE1JLX_18cranelift_frontend8frontendNtB5_15FunctionBuilder11try_use_var
-__RNvMs1_NtCsgdPmlPE1JLX_18cranelift_frontend3ssaNtB5_10SSABuilder16use_var_nonlocal
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE8grow_oneBS_
-__RNvMs1_NtCsgdPmlPE1JLX_18cranelift_frontend3ssaNtB5_10SSABuilder17run_state_machine
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEE8grow_oneB1M_
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer15list_helper_sig
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit2ir6TBlockEBF_
-__RNvMs4_NtCsgdPmlPE1JLX_18cranelift_frontend8frontendNtB5_15FunctionBuilder15seal_all_blocks
-__RNvMs1_NtCsgdPmlPE1JLX_18cranelift_frontend3ssaNtB5_10SSABuilder14seal_one_block
-__RNvMs1_NtCsgdPmlPE1JLX_18cranelift_frontend3ssaNtB5_10SSABuilder25begin_predecessors_lookup
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCsgdPmlPE1JLX_18cranelift_frontend3ssa4CallE8grow_oneBQ_
-__RNvXsa_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_9ValueDataINtNtCs8Mbv00yxnRz_4core7convert4FromNtB5_15ValueDataPackedE4from
-__RNvMsc_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph18remove_block_param
-__RNvMs2_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_8ListPoolNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE4freeB13_
-__RNvMs6_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph15change_to_alias
-__RNvMs2_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_8ListPoolNtNtCsgdPmlPE1JLX_18cranelift_frontend8variable8VariableE4freeB11_
-__RNvMs4_NtCsgdPmlPE1JLX_18cranelift_frontend8frontendNtB5_15FunctionBuilder8finalize
-__RNvMs9_NtNtCsgdPmlPE1JLX_18cranelift_frontend8frontend10safepointsNtB5_16SafepointSpiller5clear
-__RNvMs7_NtNtCsgdPmlPE1JLX_18cranelift_frontend8frontend10safepointsNtB5_16LivenessAnalysis5clear
-__RNvMse_NtCs8MhVFxlpMWQ_16cranelift_module6moduleNtB5_18ModuleDeclarations16declare_function
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRNtNtCshxvaOLs88l5_5alloc6string6StringECs8MhVFxlpMWQ_16cranelift_module
-__RNvXs3_NtNtCs8Mbv00yxnRz_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCs8MhVFxlpMWQ_16cranelift_module
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtCshxvaOLs88l5_5alloc6string6StringNtNtCs8MhVFxlpMWQ_16cranelift_module6module12FuncOrDataIdEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1s_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1w_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs8MhVFxlpMWQ_16cranelift_module6module19FunctionDeclarationE8grow_oneBQ_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner14grow_amortizedCs8MhVFxlpMWQ_16cranelift_module
-__RNvXs1_NtCslsAwG9FmMO9_13cranelift_jit7backendNtB5_9JITModuleNtNtCs8MhVFxlpMWQ_16cranelift_module6module6Module34define_function_with_control_plane
-__RNvMNtCs2EXTltNQfP1_17cranelift_codegen7contextNtB2_7Context15compile_stencil
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtCshxvaOLs88l5_5alloc5boxed3BoxDNtNtCs2EXTltNQfP1_17cranelift_codegen6timing8ProfilerEL_EEuE16get_or_init_slowNvNvNtB2r_7enabled8PROFILER27___rust_std_internal_init_fnEB2t_
-__RNvXs1_NtNtCs2EXTltNQfP1_17cranelift_codegen6timing7enabledNtB7_15DefaultProfilerNtB7_8Profiler10start_pass
-__RNvMNtCs2EXTltNQfP1_17cranelift_codegen7contextNtB2_7Context8optimize
-__RNvMNtCs2EXTltNQfP1_17cranelift_codegen7contextNtB2_7Context8legalize
-__RNvNtCs2EXTltNQfP1_17cranelift_codegen9legalizer9imm_const
-__RNvMsc_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB5_15InstructionData15mask_immediates
-__RNvXs0_NtCs2EXTltNQfP1_17cranelift_codegen6cursorQNtB5_10FuncCursorNtNtNtB7_2ir7builder16InstInserterBase17insert_built_inst
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir6layoutNtB5_6Layout11insert_inst
-__RNvXs_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB4_5InstsINtNtNtCs8Mbv00yxnRz_4core3ops5index8IndexMutNtNtB6_8entities4InstE9index_mut
-__RNvMs_NtCs2EXTltNQfP1_17cranelift_codegen9flowgraphNtB4_16ControlFlowGraph7compute
-__RNvMNtCsgSMFPVT6ofn_17cranelift_bforest4poolINtB2_8NodePoolINtNtB4_3map8MapTypesNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities4InstNtB1i_5BlockEE10alloc_nodeB1m_
-__RNvMs0_NtCsgSMFPVT6ofn_17cranelift_bforest4nodeINtB5_8NodeDataINtNtB7_3set8SetTypesNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockEE15try_leaf_insertB1p_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxDNtNtB4_3any3AnyEL_EECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen6timing7enabled18DefaultTimingTokenEBH_
-__RNvXs2_NtNtCs2EXTltNQfP1_17cranelift_codegen6timing7enabledNtB5_18DefaultTimingTokenNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellNtNtNtCs2EXTltNQfP1_17cranelift_codegen6timing7enabled9PassTimesEzE16get_or_init_slowNvNvB1R_9PASS_TIME27___rust_std_internal_init_fnEB1V_
-__RNvMs2_NtCs2EXTltNQfP1_17cranelift_codegen14dominator_treeNtB5_13DominatorTree7compute
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2EXTltNQfP1_17cranelift_codegen14dominator_tree14TraversalEventE8grow_oneBQ_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockE8grow_oneBS_
-__RNvMs_NtCsgSMFPVT6ofn_17cranelift_bforest4pathINtB4_4PathINtNtB6_3map8MapTypesNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities4InstNtB1g_5BlockEE5firstB1k_
-__RNvMs_NtCsgSMFPVT6ofn_17cranelift_bforest4pathINtB4_4PathINtNtB6_3map8MapTypesNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities4InstNtB1g_5BlockEE9next_nodeB1k_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxDNtNtB4_3any3AnyEL_EECs2EXTltNQfP1_17cranelift_codegen
-__RNvNtCs2EXTltNQfP1_17cranelift_codegen16unreachable_code26eliminate_unreachable_code
-__RNvMs0_NtCs2wcf9go0O5t_16cranelift_bitset8compoundNtB5_14CompoundBitSet19try_ensure_capacityCs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxDNtNtB4_3any3AnyEL_EECs2EXTltNQfP1_17cranelift_codegen
-__RNvNtCs2EXTltNQfP1_17cranelift_codegen20remove_constant_phis23do_remove_constant_phis
-__RNvMs6_Csa8T3MR1NSn3_7bumpaloNtB5_4Bump13with_capacity
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockNtNtB14_20remove_constant_phis12BlockSummaryE20resize_for_index_mutB14_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxDNtNtB4_3any3AnyEL_EECs2EXTltNQfP1_17cranelift_codegen
-__RNvMs6_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph19resolve_all_aliases
-__RNvMsA_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB5_15InstructionData13arguments_mut
-__RNvMs2_NtCs2EXTltNQfP1_17cranelift_codegen13loop_analysisNtB5_12LoopAnalysis7compute
-__RNvMs_NtCs2EXTltNQfP1_17cranelift_codegen14alias_analysisNtB4_13AliasAnalysis3new
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockuNtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE6insertBT_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBW_
-__RINvMsa_NtCs3fRkGY2FHAU_9hashbrown3rawNtB6_13RawTableInner22fallible_with_capacityNtNtCshxvaOLs88l5_5alloc5alloc6GlobalECs2EXTltNQfP1_17cranelift_codegen
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockNtNtBW_14alias_analysis10LastStoresEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1L_NtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBW_
-__RNvMNtCs2EXTltNQfP1_17cranelift_codegen14alias_analysisNtB2_10LastStores6update
-__RNCNvMs_NtCs2EXTltNQfP1_17cranelift_codegen14alias_analysisNtB6_13AliasAnalysis26compute_block_input_statess_0B8_
-__RNvMs0_NtCs2EXTltNQfP1_17cranelift_codegen6egraphNtB5_10EgraphPass3run
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueBY_E20resize_for_index_mutB14_
-__RNvNtCs2EXTltNQfP1_17cranelift_codegen15inst_predicates18is_pure_for_egraph
-__RINvNtNtCs2EXTltNQfP1_17cranelift_codegen4opts14generated_code29constructor_simplify_skeletonNtB4_11IsleContextINtCsfNVduRRWYaN_8smallvec8SmallVecANtB2_26SkeletonInstSimplificationj5_EEB6_
-__RNvMsA_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB5_15InstructionData9arguments
-__RNvMs_NtNtCs2EXTltNQfP1_17cranelift_codegen6egraph4costNtB4_4Cost9of_opcode
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2EXTltNQfP1_17cranelift_codegen16take_and_replace14TakeAndReplaceNtNtBG_6egraph11OptimizeCtxINtCsfNVduRRWYaN_8smallvec8SmallVecANtNtNtBG_4opts14generated_code26SkeletonInstSimplificationj5_ENCNvMs_B1N_B1L_22simplify_skeleton_inst0EEBG_
-__RNvMNtCs2EXTltNQfP1_17cranelift_codegen14alias_analysisNtB2_10LastStores14get_last_store
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapNtNtCs2EXTltNQfP1_17cranelift_codegen14alias_analysis9MemoryLocTNtNtNtBR_2ir8entities4InstNtB1R_5ValueENtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE6insertBR_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtCs2EXTltNQfP1_17cranelift_codegen14alias_analysis9MemoryLocTNtNtNtBU_2ir8entities4InstNtB1U_5ValueEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1R_NtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBU_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2EXTltNQfP1_17cranelift_codegen6egraph11OptimizeCtxEBF_
-__RNvMs_NtCs2EXTltNQfP1_17cranelift_codegen6egraphNtB4_11OptimizeCtx17insert_pure_enode
-__RNvMsb_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph12ctrl_typevar
-__RINvXs2_NtCs2EXTltNQfP1_17cranelift_codegen6egraphNtB6_10GVNContextINtNtB8_7ctxhash7CtxHashTNtNtNtB8_2ir5types4TypeNtNtB1w_12instructions15InstructionDataEE8ctx_hashNtCsfoFpBBKvNFT_10rustc_hash8FxHasherEB8_
-__RINvNtNtCs2EXTltNQfP1_17cranelift_codegen4opts14generated_code20constructor_simplifyNtB4_11IsleContextINtCsfNVduRRWYaN_8smallvec8SmallVecANtNtNtB6_2ir8entities5Valuej5_EEB6_
-__RNvXs2_NtCs2EXTltNQfP1_17cranelift_codegen4optsNtB5_16InstDataEtorIterNtNtB5_14generated_code11ContextIter4next
-__RNvMs6_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph9value_def
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBW_
-__RINvXss_CsfNVduRRWYaN_8smallvecINtB6_8SmallVecANtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5Valuej5_EINtNtNtNtCs8Mbv00yxnRz_4core4iter6traits7collect6ExtendBJ_E6extendINtNtB1R_6option6OptionBJ_EEBP_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2EXTltNQfP1_17cranelift_codegen16take_and_replace14TakeAndReplaceNtNtBG_6egraph11OptimizeCtxINtCsfNVduRRWYaN_8smallvec8SmallVecANtNtNtBG_2ir8entities5Valuej5_ENCNvMs_B1N_B1L_19optimize_pure_enode0EEBG_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir6layoutNtB5_6Layout11remove_inst
-__RNvMsA_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB5_15InstructionData2eq
-__RNvMsA_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB5_15InstructionData15typevar_operand
-__RNvXs5_NtCs2EXTltNQfP1_17cranelift_codegen4optsNtB5_18MaybeUnaryEtorIterNtNtB5_14generated_code11ContextIter4next
-__RINvNtNtCs2EXTltNQfP1_17cranelift_codegen4opts14generated_code18constructor_truthyNtB4_11IsleContextINtB2_18ContextIterWrapperINtCsfNVduRRWYaN_8smallvec8SmallVecANtNtNtB6_2ir8entities5Valuej8_EB1i_EEB6_
-__RNvMs0_NtNtCs2EXTltNQfP1_17cranelift_codegen6egraph9elaborateNtB5_10Elaborator9elaborate
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtCs2EXTltNQfP1_17cranelift_codegen10traversals5EventNtNtNtBR_2ir8entities5BlockEE8grow_oneBR_
-__RNvXs_NtCs2EXTltNQfP1_17cranelift_codegen10traversalsNtB4_7DfsIterNtNtNtNtCs8Mbv00yxnRz_4core4iter6traits8iterator8Iterator4next
-__RNvMsb_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph11inst_values
-__RNvXs_NtNtNtCs8Mbv00yxnRz_4core4iter8adapters5chainINtB4_5ChainIBO_INtNtB6_6copied6CopiedINtNtNtBa_5slice4iter4IterNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueEEINtNtB6_7flatten7FlatMapIB1q_NtNtB1T_12instructions9BlockCallEINtNtB6_10filter_map9FilterMapINtNtB6_3map3MapB1p_NCNvMB3h_B3f_4args0ENCNCNvMsb_NtB1T_3dfgNtB54_13DataFlowGraph11inst_values00ENCB4Y_0EEIB2N_INtNtBa_6option8IntoIterNtB1R_14ExceptionTableEIB3N_IB1q_NtNtB1T_15exception_table26InternalExceptionTableItemENCNvMB70_NtB70_18ExceptionTableData8contexts0ENCB4Y_s_0EENtNtNtB8_6traits8iterator8Iterator9size_hintB1V_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen6egraph9elaborate15BlockStackEntryE8grow_oneBS_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen6egraph9elaborate14ElabStackEntryE8grow_oneBS_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen6egraph9elaborate15ElaboratedValueE8grow_oneBS_
-__RNvMs0_NtNtCs2EXTltNQfP1_17cranelift_codegen6egraph9elaborateNtB5_10Elaborator15maybe_remat_arg
-__RINvMs0_NtCs2EXTltNQfP1_17cranelift_codegen15scoped_hash_mapINtB6_13ScopedHashMapNtNtNtB8_2ir8entities5ValueNtNtNtB8_6egraph9elaborate15ElaboratedValueE27insert_if_absent_with_depthNtNtB8_7ctxhash7NullCtxEB8_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTTNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockNtBT_5ValueEB1M_EE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1M_NtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBX_
-__RNvMsb_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph10clone_inst
-__RNvMsA_NtNtCs2EXTltNQfP1_17cranelift_codegen2ir12instructionsNtB5_15InstructionData10deep_clone
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueNtNtNtB14_6egraph9elaborate9BestEntryE20resize_for_index_mutB14_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen6egraph9elaborate10ElaboratorEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2EXTltNQfP1_17cranelift_codegen14alias_analysis13AliasAnalysisEBF_
-__RNvXs_NtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch64NtB4_14AArch64BackendNtB6_9TargetIsa16compile_function
-__RINvMs5_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abiNtB6_6SigSet13from_func_sigNtNtNtNtBa_3isa7aarch643abi18AArch64MachineDepsEBa_
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch643abiNtB5_18AArch64MachineDepsNtNtNtBb_8machinst3abi14ABIMachineSpec16compute_arg_locs
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644instNtNtNtNtB7_5lower4isle14generated_code5MInstNtNtBb_8machinst8MachInst11rc_for_type
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7extfunc9SignatureNtNtNtBW_8machinst3abi3SigEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1O_NtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBW_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities6SigRefINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtNtB14_8machinst3abi3SigEE20resize_for_index_mutB14_
-__RNvMs5_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abiNtB5_6SigSet21abi_sig_for_signature
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2EXTltNQfP1_17cranelift_codegen6result12CodegenErrorEBF_
-__RNvNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abi36ensure_struct_return_ptr_is_returned
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlocklE20resize_for_index_mutB14_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockINtNtNtCs8Mbv00yxnRz_4core3ops5range5RangejEE20resize_for_index_mutB14_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst10blockorder12LoweredBlockE8grow_oneBS_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5BlockNtNtCs2jIxScAqd7x_9regalloc25index5BlockE20resize_for_index_mutB14_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst10blockorder12LoweredBlockNtNtCs2jIxScAqd7x_9regalloc25index5BlockEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B22_NtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBW_
-__RNvMs8_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeNtB5_14VCodeConstants13with_capacity
-__RNvMs7_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_13VRegAllocatorNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE5allocB9_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2EXTltNQfP1_17cranelift_codegen6result12CodegenErrorEBF_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueINtNtNtB14_8machinst9valueregs9ValueRegsNtNtB1Y_3reg3RegEE20resize_for_index_mutB14_
-__RNvNtCs2EXTltNQfP1_17cranelift_codegen15inst_predicates24has_lowering_side_effect
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities4InstNtNtNtBW_8machinst5lower9InstColorEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1K_NtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBW_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities4InstyEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_yNtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBW_
-__RINvNtNtNtCs8Mbv00yxnRz_4core4iter8adapters5chain17and_then_or_clearINtB2_5ChainINtNtB4_6copied6CopiedINtNtNtB8_5slice4iter4IterNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueEEINtNtB4_7flatten7FlatMapIB1D_NtNtB26_12instructions9BlockCallEINtNtB4_10filter_map9FilterMapINtNtB4_3map3MapB1C_NCNvMB3u_B3s_4args0ENCNCNvMsb_NtB26_3dfgNtB5h_13DataFlowGraph11inst_values00ENCB5b_0EEB22_NvYB14_NtNtNtB6_6traits8iterator8Iterator4nextEB28_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueNtNtNtB14_8machinst5lower13ValueUseStateE20resize_for_index_mutB14_
-__RINvMs1_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5lowerINtB6_5LowerNtNtNtNtNtNtBa_3isa7aarch645lower4isle14generated_code5MInstE5lowerNtB1g_14AArch64BackendEBa_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code17constructor_lowerINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code24constructor_lower_returnINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs3_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_10EntityListNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE3getB16_
-__RNvMs3_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5lowerINtB5_5LowerNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE17put_value_in_regsB9_
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context10gen_return
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtCsfNVduRRWYaN_8smallvec8IntoIterANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj4_EEB1n_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstE8grow_oneBY_
-__RNvMs1_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5lowerINtB5_5LowerNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE14finish_ir_instB9_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtNtB4_4iter8adapters3rev3RevINtNtNtCshxvaOLs88l5_5alloc3vec5drain5DrainNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstEEEB22_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code15constructor_immINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code24constructor_operand_sizeINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs7_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_13VRegAllocatorNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE25alloc_with_deferred_errorB9_
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context4emit
-__RNvXsh_NtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_codeNtB5_5MInstNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstEBN_
-__RNvMs7_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_13VRegAllocatorNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE14set_vreg_aliasB9_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code17constructor_amodeINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code32constructor_amode_no_more_iconstINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs4_NtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4immsNtB5_12UImm12Scaled14maybe_from_i64
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code28constructor_emit_side_effectINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code18SideEffectNoResultEBN_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code27constructor_aarch64_uload64INtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvXNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lowerNtB4_14AArch64BackendNtNtNtB8_8machinst5lower12LowerBackend12lower_branch
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code33constructor_lower_icmp_into_flagsINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code22constructor_lower_icmpINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context10fits_in_16
-__RNvXsb_NtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_codeNtB5_13ProducesFlagsNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RNvXsh_NtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_codeNtB5_5MInstNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code13ProducesFlagsEBN_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code34constructor_with_flags_side_effectINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code13ConsumesFlagsEBN_
-__RNvMs1_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5lowerINtB5_5LowerNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE28lower_branch_blockparam_argsB9_
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context15gen_call_output
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context7abi_sig
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecAINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst9valueregs9ValueRegsNtNtBN_3reg3RegEj2_E8try_growBP_
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context13gen_call_args
-__RNvMs9_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abiINtB5_6CalleeNtNtNtNtB9_3isa7aarch643abi18AArch64MachineDepsE13gen_call_argsB9_
-__RNCNvMs9_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abiINtB7_6CalleeNtNtNtNtBb_3isa7aarch643abi18AArch64MachineDepsE13gen_call_argss_0Bb_
-__RNvMs5_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abiNtB5_6SigSet11get_ret_arg
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context13gen_call_rets
-__RNvMs9_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abiINtB5_6CalleeNtNtNtNtB9_3isa7aarch643abi18AArch64MachineDepsE13gen_call_retsB9_
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context17gen_call_ind_info
-__RNvXsd_NtCshxvaOLs88l5_5alloc5boxedINtB5_3BoxINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abi8CallInfoNtNtBM_3reg3RegEENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBO_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abi8CallInfoNtNtBG_3reg3RegEEBI_
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context33move_wide_const_from_inverted_u64
-__RNvMs6_NtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4immsNtB5_8ImmLogic14maybe_from_u64
-__RNvMsd_NtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4argsNtB5_11OperandSize7from_ty
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context18load_constant_full
-__RNvMs9_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abiINtB5_6CalleeNtNtNtNtB9_3isa7aarch643abi18AArch64MachineDepsE20gen_copy_arg_to_regsB9_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abi7ArgPairE8grow_oneBS_
-__RINvNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst20aarch64_get_operandsINtNtNtB8_8machinst3reg16OperandCollectorNCNvMs1_NtB1n_5vcodeINtB25_12VCodeBuilderNtNtNtNtB4_5lower4isle14generated_code5MInstE16collect_operands0EEB8_
-__RNvXsl_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3regINtB5_16OperandCollectorNCNvMs1_NtB7_5vcodeINtB1p_12VCodeBuilderNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE16collect_operands0ENtB5_14OperandVisitor11add_operandB9_
-__RINvNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst15memarg_operandsINtNtNtB8_8machinst3reg16OperandCollectorNCNvMs1_NtB1i_5vcodeINtB20_12VCodeBuilderNtNtNtNtB4_5lower4isle14generated_code5MInstE16collect_operands0EEB8_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtCs2jIxScAqd7x_9regalloc25index4InstNtBU_7PRegSetEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1t_NtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0ECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode13VRegAllocatorNtNtNtNtNtNtBI_3isa7aarch645lower4isle14generated_code5MInstEEBI_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstEEB1k_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxDNtNtB4_3any3AnyEL_EECs2EXTltNQfP1_17cranelift_codegen
-__RINvCs2jIxScAqd7x_9regalloc23runINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtBB_3isa7aarch645lower4isle14generated_code5MInstEEBB_
-__RINvMNtCs2jIxScAqd7x_9regalloc23cfgNtB3_7CFGInfo4initINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtBW_3isa7aarch645lower4isle14generated_code5MInstEEBW_
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function11block_succsB9_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2jIxScAqd7x_9regalloc25index5BlockE8grow_oneCs2EXTltNQfP1_17cranelift_codegen
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function11block_predsB9_
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function11block_insnsB9_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecmE8grow_oneCs2EXTltNQfP1_17cranelift_codegen
-__RNvMNtCs2jIxScAqd7x_9regalloc23ionINtNtB2_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB16_3isa7aarch645lower4isle14generated_code5MInstEE4initB16_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures8VRegDataE8grow_oneBS_
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs2jIxScAqd7x_9regalloc2
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function13inst_operandsB9_
-__RNvMs3_NtNtCshxvaOLs88l5_5alloc11collections9vec_dequeINtB5_8VecDequeNtNtCs2jIxScAqd7x_9regalloc25index5BlockE4growCs2EXTltNQfP1_17cranelift_codegen
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function9is_branchB9_
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function12block_paramsB9_
-__RNvMs0_NtCs2jIxScAqd7x_9regalloc28indexsetNtB5_8IndexSet10union_with
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function18branch_blockparamsB9_
-__RNvMs0_NtCs2jIxScAqd7x_9regalloc28indexsetNtB5_8IndexSet8is_empty
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function13inst_clobbersB9_
-__RNvMs0_NtNtCs2jIxScAqd7x_9regalloc23ion10liverangesINtNtB7_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1n_3isa7aarch645lower4isle14generated_code5MInstEE21add_liverange_to_vregB1n_
-__RNvMs6_NtNtCs2jIxScAqd7x_9regalloc23ion15data_structuresNtB5_10LiveRanges3add
-__RNvXsw_Cs2jIxScAqd7x_9regalloc2NtB5_4BumpNtNtNtCs7WGKQJlwkvb_14allocator_api26stable5alloc9Allocator8allocate
-__RNvMs7_Csa8T3MR1NSn3_7bumpaloNtB5_4Bump17alloc_layout_slowCs2jIxScAqd7x_9regalloc2
-__RNvMs0_NtNtCs2jIxScAqd7x_9regalloc23ion10liverangesINtNtB7_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1n_3isa7aarch645lower4isle14generated_code5MInstEE25insert_use_into_liverangeB1n_
-__RNvMsi_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB5_8BTreeMapNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12LiveRangeKeyNtB19_14LiveRangeIndexE6insertCs2EXTltNQfP1_17cranelift_codegen
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures18MultiFixedRegFixupE8grow_oneCs2EXTltNQfP1_17cranelift_codegen
-__RNvMs7_Csa8T3MR1NSn3_7bumpaloNtB5_4Bump4growCs2jIxScAqd7x_9regalloc2
-__RNvMs7_NtNtCs2jIxScAqd7x_9regalloc23ion15data_structuresNtB5_11LiveBundles3add
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion5mergeINtNtB4_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1e_3isa7aarch645lower4isle14generated_code5MInstEE13merge_bundlesB1e_
-__RNvMs_NtNtCs7WGKQJlwkvb_14allocator_api26stable3vecINtB4_3VecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures18LiveRangeListEntryNtB13_4BumpE6insertCs2EXTltNQfP1_17cranelift_codegen
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion5mergeINtNtB4_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1e_3isa7aarch645lower4isle14generated_code5MInstEE23merge_bundle_propertiesB1e_
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion7processINtNtB4_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1g_3isa7aarch645lower4isle14generated_code5MInstEE27recompute_bundle_propertiesB1g_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures14PrioQueueEntryE8grow_oneCs2EXTltNQfP1_17cranelift_codegen
-__RNvMNtCs2jIxScAqd7x_9regalloc23ionINtNtB2_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB16_3isa7aarch645lower4isle14generated_code5MInstEE3runB16_
-__RNvMs0_NtNtCs2jIxScAqd7x_9regalloc23ion11requirementINtNtB7_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1o_3isa7aarch645lower4isle14generated_code5MInstEE19compute_requirementB1o_
-__RNvMs_NtNtCs2jIxScAqd7x_9regalloc23ion13reg_traversalNtB4_16RegTraversalIter3new
-__RNvXs0_NtNtCs2jIxScAqd7x_9regalloc23ion13reg_traversalNtB5_16RegTraversalIterNtNtNtNtCs8Mbv00yxnRz_4core4iter6traits8iterator8Iterator4next
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion7processINtNtB4_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1g_3isa7aarch645lower4isle14generated_code5MInstEE29try_to_allocate_bundle_to_regB1g_
-__RINvMsi_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB6_8BTreeMapNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12LiveRangeKeyNtB1a_14LiveRangeIndexE5rangeB18_INtNtNtCs8Mbv00yxnRz_4core3ops5range9RangeFromB18_EECs2EXTltNQfP1_17cranelift_codegen
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion7processINtNtB4_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1g_3isa7aarch645lower4isle14generated_code5MInstEE24split_and_requeue_bundleB1g_
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion7processINtNtB4_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1g_3isa7aarch645lower4isle14generated_code5MInstEE26get_or_create_spill_bundleB1g_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures15LiveBundleIndexE8grow_oneCs2EXTltNQfP1_17cranelift_codegen
-__RINvMs_NtNtNtCshxvaOLs88l5_5alloc11collections5btree6removeINtNtB7_4node6HandleINtBY_7NodeRefNtNtBY_6marker3MutNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12LiveRangeKeyNtB1N_14LiveRangeIndexNtB1v_4LeafENtB1v_2KVE14remove_leaf_kvNCNvMs5_NtNtB7_3map5entryINtB3U_13OccupiedEntryB1L_B2O_E9remove_kv0NtNtBb_5alloc6GlobalECs2EXTltNQfP1_17cranelift_codegen
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12InsertedMoveE8grow_oneBS_
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function6is_retB9_
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion15redundant_movesNtB2_23RedundantMoveEliminator5clear
-__RNvMNtCs2jIxScAqd7x_9regalloc25movesINtB2_13ParallelMovesINtNtCs8Mbv00yxnRz_4core6option6OptionNtB4_4VRegEE7resolveCs2EXTltNQfP1_17cranelift_codegen
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion15redundant_movesNtB2_23RedundantMoveEliminator12process_move
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion15redundant_movesNtB2_23RedundantMoveEliminator11clear_alloc
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2jIxScAqd7x_9regalloc23ion15redundant_moves23RedundantMoveEliminatorECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2jIxScAqd7x_9regalloc23cfg7CFGInfoECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2jIxScAqd7x_9regalloc23cfg10CFGInfoCtxECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2jIxScAqd7x_9regalloc28indexset8IndexSetEECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures10LiveRangesECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures11LiveBundlesECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures5VRegsECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures8PRegDataEECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures13SpillSlotDataEECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures14SpillSetRangesEECs2EXTltNQfP1_17cranelift_codegen
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcNtCsa8T3MR1NSn3_7bumpalo4BumpE9drop_slowCs2jIxScAqd7x_9regalloc2
-__RNvMse_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer9MachLabelj10_E6resizeBO_
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644instNtNtNtNtB7_5lower4isle14generated_code5MInstNtNtBb_8machinst8MachInst23is_included_in_clobbers
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch643abiNtB5_18AArch64MachineDepsNtNtNtBb_8machinst3abi14ABIMachineSpec20compute_frame_layout
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emitNtB2_9EmitStateINtNtBa_8machinst17MachInstEmitStateNtNtNtNtB6_5lower4isle14generated_code5MInstE3new
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch643abiNtB5_18AArch64MachineDepsNtNtNtBb_8machinst3abi14ABIMachineSpec24gen_prologue_frame_setup
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj4_E21reserve_one_uncheckedBU_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj4_E8try_growBU_
-__RINvXss_CsfNVduRRWYaN_8smallvecINtB6_8SmallVecANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj4_EINtNtNtNtCs8Mbv00yxnRz_4core4iter6traits7collect6ExtendBJ_E6extendBt_EBV_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtCsfNVduRRWYaN_8smallvec8IntoIterANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj4_EEB1n_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstEEB1k_
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch643abiNtB5_18AArch64MachineDepsNtNtNtBb_8machinst3abi14ABIMachineSpec16gen_clobber_save
-__RINvXss_CsfNVduRRWYaN_8smallvecINtB6_8SmallVecANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj4_EINtNtNtNtCs8Mbv00yxnRz_4core4iter6traits7collect6ExtendBJ_E6extendIBu_ABJ_j10_EEBV_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtCsfNVduRRWYaN_8smallvec8IntoIterANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj10_EEB1n_
-__RNCNvMs2_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB7_5VCodeNtNtNtNtNtNtBb_3isa7aarch645lower4isle14generated_code5MInstE4emit0Bb_
-__RNvXs1_NtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emitNtNtNtNtB9_5lower4isle14generated_code5MInstNtNtBd_8machinst12MachInstEmit4emit
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE4put4B9_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE10add_unwindB9_
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit13enc_ldst_pair
-__RNvMs0_NtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4immsNtB5_11SImm7Scaled4bits
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit14machreg_to_gpr
-__RNvMNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3regNtB2_3Reg5class
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit18enc_arith_rr_imm12
-__RNvMs_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644instNtNtNtNtB6_5lower4isle14generated_code5MInst8mem_type
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit12mem_finalize
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtCsfNVduRRWYaN_8smallvec8IntoIterANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj4_EEB1n_
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit14enc_ldst_simm9
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstEBN_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE10bind_labelB9_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE17optimize_branchesB9_
-__RINvMsr_Cs2jIxScAqd7x_9regalloc2NtB6_6Output21block_insts_and_editsINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1a_3isa7aarch645lower4isle14generated_code5MInstEEB1a_
-__RNCNvMs2_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB7_5VCodeNtNtNtNtNtNtBb_3isa7aarch645lower4isle14generated_code5MInstE4emits0_0Bb_
-__RNvMsr_Cs2jIxScAqd7x_9regalloc2NtB5_6Output11inst_allocs
-__RNvYNCNvMs2_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtBa_5VCodeNtNtNtNtNtNtBe_3isa7aarch645lower4isle14generated_code5MInstE4emits1_0NtNtBc_3reg18OperandVisitorImpl9reg_fixedBe_
-__RINvNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst15memarg_operandsNCNvMs2_NtNtB8_8machinst5vcodeINtB1l_5VCodeNtNtNtNtB4_5lower4isle14generated_code5MInstE4emits1_0EB8_
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit15enc_ldst_uimm12
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE19use_label_at_offsetB9_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE17add_uncond_branchB9_
-__RNvMs5_NtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4argsNtB5_12BranchTarget19as_offset26_or_zero
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE13island_neededB9_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE20truncate_last_branchB9_
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit13enc_move_wide
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit8enc_movk
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644instNtNtNtNtB7_5lower4isle14generated_code5MInstNtNtBb_8machinst8MachInst8gen_move
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit13enc_arith_rrr
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstEBN_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE13add_call_siteB9_
-__RINvMsb_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abiINtB6_8CallInfoNtNtB8_3reg3RegE17emit_retval_loadsNtNtNtNtBa_3isa7aarch643abi18AArch64MachineDepsNCNvXs1_NtNtB1M_4inst4emitNtNtNtNtB1M_5lower4isle14generated_code5MInstNtB8_12MachInstEmit4emits0_0NCB2v_s1_0EBa_
-__RNvMs4_NtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4argsNtB5_10CondBrKind6invert
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit18enc_conditional_br
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE15add_cond_branchB9_
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch643abiNtB5_18AArch64MachineDepsNtNtNtBb_8machinst3abi14ABIMachineSpec19gen_clobber_restore
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtCsfNVduRRWYaN_8smallvec8IntoIterANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj4_EEB1n_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer10MachBranchEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit9EmitStateEBL_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer21MachBufferFrameLayoutEEB13_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE24emit_island_maybe_forcedB9_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE18should_apply_fixupB9_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE12handle_fixupB9_
-__RNvXs2_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644instNtB5_8LabelUseNtNtBb_8machinst16MachInstLabelUse5patch
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtCsfNVduRRWYaN_8smallvec8IntoIterAINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer14MachLabelFixupNtNtNtNtNtNtB1i_3isa7aarch645lower4isle14generated_code5MInstEj10_EEB1i_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtCsfNVduRRWYaN_8smallvec8IntoIterANtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer13MachLabelTrapj10_EEB1h_
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644instNtNtNtNtB7_5lower4isle14generated_code5MInstNtNtBb_8machinst8MachInst13gen_nop_units
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtBI_3isa7aarch645lower4isle14generated_code5MInstEEBI_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstEEB1k_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst10blockorder18BlockLoweringOrderEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abi6CalleeNtNtNtNtBI_3isa7aarch643abi18AArch64MachineDepsEEBI_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7extfunc8AbiParamEEB1e_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode14VCodeConstantsEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs7fImLwyQvo9_16cranelift_entity7primary10PrimaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode13VCodeConstantNtB1y_17VCodeConstantDataEEB1C_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abi6SigSetEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7extfunc8AbiParamEEB1e_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abi6ABIArgEEB1e_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs2jIxScAqd7x_9regalloc26OutputECs2EXTltNQfP1_17cranelift_codegen
-__RNvMs_NtCs2EXTltNQfP1_17cranelift_codegen8machinstINtB4_16CompiledCodeBaseNtNtB4_6buffer7StencilE12apply_params
-__RNvMs0_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_19MachBufferFinalizedNtB5_7StencilE17apply_base_srcloc
-__RNvXs_NtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch64NtB4_14AArch64BackendNtB6_9TargetIsa18function_alignment
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtCs8MhVFxlpMWQ_16cranelift_module6module6FuncIdINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCslsAwG9FmMO9_13cranelift_jit13compiled_blob12CompiledBlobEE20resize_for_index_mutB2s_
-__RINvNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECslsAwG9FmMO9_13cranelift_jit
-__RNvMs5_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11finish_growCslsAwG9FmMO9_13cranelift_jit
-__RNvXs3_NtNtCslsAwG9FmMO9_13cranelift_jit6memory6systemNtB5_20SystemMemoryProviderNtB7_17JITMemoryProvider8allocate
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCslsAwG9FmMO9_13cranelift_jit6memory6system6PtrLenE8grow_oneBS_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once9call_onceNCNvNtCs2O21nn49adY_6region4page4size0E0CslsAwG9FmMO9_13cranelift_jit
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTjjNtNtCs8MhVFxlpMWQ_16cranelift_module6module6FuncIdEE8grow_oneCslsAwG9FmMO9_13cranelift_jit
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs8MhVFxlpMWQ_16cranelift_module6module6FuncIdE8grow_oneCslsAwG9FmMO9_13cranelift_jit
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7extfunc9SignatureEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7extfunc8AbiParamEEB1e_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecIBC_hEEECs2EXTltNQfP1_17cranelift_codegen
-__RNvMs0_NtCslsAwG9FmMO9_13cranelift_jit7backendNtB5_9JITModule20finalize_definitions
-__RNvXs_NtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch64NtB4_14AArch64BackendNtB6_9TargetIsa9isa_flags
-__RNvXs3_NtNtCslsAwG9FmMO9_13cranelift_jit6memory6systemNtB5_20SystemMemoryProviderNtB7_17JITMemoryProvider8finalize
-__RNvMs0_NtCslsAwG9FmMO9_13cranelift_jit7backendNtB5_9JITModule22get_finalized_function
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEENtNtBK_5clone5Clone5cloneB1l_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir11GlobalGuardENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12ListLoopMetaENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12IterLoopMetaENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir13CompSavedMetaENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12AttrSiteMetaENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir14MethodSiteMetaENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir9StrMethodENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir13MathGuardMetaENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvXsa_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir8OsrEntryENtNtCs8Mbv00yxnRz_4core5clone5Clone5cloneBJ_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit6engine14is_scalar_leaf
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit2ir5TFuncEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir6TBlockEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit2ir6TBlockEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir11GlobalGuardEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12AttrSiteMetaEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit2ir12AttrSiteMetaEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECseWU5rFlE7Md_11weavepy_jit
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir14MethodSiteMetaEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir13MathGuardMetaEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir8OsrEntryEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit7analyze9PathArenaECs2D882usVKTN_10weavepy_vm
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecTNtNtB6_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters10filter_map9FilterMapINtNtNtB2K_5slice4iter4IterNtNtCseWU5rFlE7Md_11weavepy_jit2ir11GlobalGuardENCNvMs1_NtB1r_5tier2NtB4Y_8JitState12get_compileds1_0EE9from_iterB1r_
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtB2p_5slice4iter4IterNtNtB8_6string6StringENCNvMs1_NtB17_5tier2NtB3Z_8JitState12get_compileds2_0EE9from_iterB17_
-__RNCNCNvNtCs2D882usVKTN_10weavepy_vm5tier212warm_compile0s3_0B7_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215attr_site_guard
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215attr_site_guard0B5_
-__RNCNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB7_8JitState12get_compileds4_0B9_
-__RNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB5_8JitState16native_table_for
-__RNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB5_8JitState23method_native_table_for
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs2D882usVKTN_10weavepy_vm
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_mod5build
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11marshal_mod8register
-__RNCINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB4_12int_to_bytesEs_0B6_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins12int_to_bytesINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1C_EEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins13byteorder_str
-__RNvNtNtCsfGIcTr55AEy_10num_bigint7biguint7convert20to_bitwise_digits_le
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNvNtCs2D882usVKTN_10weavepy_vm8builtins12int_to_bytesE9drop_slowBK_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7imp_mod22imp_extension_suffixes
-__RNvCs2D882usVKTN_10weavepy_vm14spec_parent_of
-__RNvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6Object12interned_str
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexINtNtNtNtB7_11collections4hash3set7HashSetNtNtCs2D882usVKTN_10weavepy_vm12shared_value9SharedStrEEE10initializeNCINvB2_11get_or_initNCNvMs1j_NtB26_6objectNtB3D_6Object12interned_str0E0zEB26_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4time5build
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4time1b
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4time16struct_time_type
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4time16compute_timezone
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4time16compute_timezone0B7_
-__RNvMNtNtCsgVNPin0H17l_6chrono6offset3utcNtB2_3Utc3now
-__RNvMNtNtCsgVNPin0H17l_6chrono5naive4dateNtB2_9NaiveDate25from_num_days_from_ce_opt
-__RNvMsW_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_11BoundMethod16is_pymethod_type
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins10dict_clearINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins10dict_clear
-__RNvXse_NtNtCshxvaOLs88l5_5alloc3vec9into_iterINtB5_8IntoIterTNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBZ_6ObjectEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropB11_
-__RNvMs2_NtCsblfCQaHOzq5_8indexmap3mapINtB5_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBP_6ObjectNtNtBR_8fasthash13FxBuildHasherE5entryBR_
-__RNvMs_NtCsblfCQaHOzq5_8indexmap5innerINtB4_4CoreNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBM_6ObjectE13insert_uniqueBO_
-__RNvCs2D882usVKTN_10weavepy_vm13bind_int_args
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11do_int_call
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins12b_int_compat
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins16parse_int_string
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins27transform_decimal_and_space
-__RNvMsr_NtCs8Mbv00yxnRz_4core3numx16from_ascii_radix
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4time17time_monotonic_ns
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockNtNtB7_4time7InstantE10initializeNCINvB2_11get_or_initNvMBV_BT_3nowE0zECs2D882usVKTN_10weavepy_vm
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtBc_4time7InstantE10initializeNCINvB1a_11get_or_initNvMB1I_B1G_3nowE0zE0E0Cs2D882usVKTN_10weavepy_vm
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21percent_mod_left_slot
-__RNvCs2D882usVKTN_10weavepy_vm24percent_args_need_bridge
-__RNvCs2D882usVKTN_10weavepy_vm19percent_format_with
-__RINvMs3_NtCshxvaOLs88l5_5alloc3stre7replaceAcj2_ECs2D882usVKTN_10weavepy_vm
-__RNvCs2D882usVKTN_10weavepy_vm10format_int
-__RNvCs2D882usVKTN_10weavepy_vm16group_int_digits
-__RNvCs2D882usVKTN_10weavepy_vm19int_precision_apply
-__RNvCs2D882usVKTN_10weavepy_vm9with_sign
-__RNvCs2D882usVKTN_10weavepy_vm15apply_alignment
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter8do_print
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter8do_prints_0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4math5build
-__RNvCs2D882usVKTN_10weavepy_vm20checked_repeat_count
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4math7builtin
-__RINvCs2D882usVKTN_10weavepy_vm13try_alloc_vecNtNtB2_6object6ObjectEB2_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze17const_slice_shape
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15bin_result_type
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins7b_bytesINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib14operator_accel5build
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins10b_bytes_kw
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13itertools_mod5build
-__RNvYNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13itertools_mod31register_classmethod_descriptorINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib14operator_accel5build0B7_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins21bytes_from_source_obj
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib14datetime_accel5build
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13itertools_mod31register_classmethod_descriptor
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB4_6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit2ir6TBlockEEEB1y_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6result6ResultNtNtCs2D882usVKTN_10weavepy_vm6object10PyIteratorNtNtB11_5error12RuntimeErrorEEB11_
-__RNvYFG_RL0_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtNtCs8Mbv00yxnRz_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins29bytes_from_iterable_reentrant
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib14datetime_accel15timedelta_parts
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins15byte_item_value
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RNvCs2D882usVKTN_10weavepy_vm9float_pow
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12AttrSiteMetaEEB1c_
-__RINvCs2D882usVKTN_10weavepy_vm13try_alloc_vechEB2_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB4_6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit2ir14MethodSiteMetaEEEB1y_
-__RNvNtCs2D882usVKTN_10weavepy_vm14descr_registry10descr_type
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm5types12InstancePlanE9drop_slowBK_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11b_enumerateINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm14descr_registry9DescrMetaEBF_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11b_enumerate
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins22coexist_builtin_dunder
-__RNvCs2D882usVKTN_10weavepy_vm16class_dunder_new
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs8Mbv00yxnRz_4core4cell4CelljEE4withNCNvCs2D882usVKTN_10weavepy_vm13is_object_new0jEB1E_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir13MathGuardMetaEEB1c_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmNtNtCseWU5rFlE7Md_11weavepy_jit2ir14ResolvedGlobalEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBV_
-__RNvCs2D882usVKTN_10weavepy_vm17is_union_eligible
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter22deopt_store_attr_taken
-__RNvCs2D882usVKTN_10weavepy_vm18is_union_initiator
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit7analyze4PlanEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir11GlobalGuardEEB1c_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib14datetime_accel11date_fields
-__RNvCs2D882usVKTN_10weavepy_vm17make_pep604_union
-__RNCNCNvNtCs2D882usVKTN_10weavepy_vm5tier212warm_compile0s6_0B7_
-__RNCNCNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB9_8JitState12get_compileds0_00Bb_
-__RNCNvCs2D882usVKTN_10weavepy_vm17make_pep604_union0B3_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter25push_pending_caller_shell
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier216probe_param_lane
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib14datetime_accel11time_fields
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze7pop_val
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB1b_6ObjectEEEB1d_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native7install
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter12do_hash_call
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze9lower_bin
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier219resolve_plain_dicts
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11hash_object
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215classify_global
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRThReEECs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTxNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE8grow_oneBS_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier216probe_class_ctor
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTThReENtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_BV_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBZ_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTxNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB1e_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier222probe_class_ctor_shape
-__RNvMNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_nativeNtB2_5Names3new
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjTmmENtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCseWU5rFlE7Md_11weavepy_jit
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native8interned
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native5build
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native5build0B7_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjTmmEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECseWU5rFlE7Md_11weavepy_jit
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjTmmmENtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCseWU5rFlE7Md_11weavepy_jit
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer14div_guard_cond
-__RNvNtCs2D882usVKTN_10weavepy_vm13leaf_builtins4push
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjTmmmEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECseWU5rFlE7Md_11weavepy_jit
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjmNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCseWU5rFlE7Md_11weavepy_jit
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyEBG_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTINtNtB7_4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm6object9BuiltinFnEINtNtCs8Mbv00yxnRz_4core6option6OptionFG_RL0_SNtB17_6ObjectEIB1S_INtNtB1W_6result6ResultB2B_NtNtB19_5error12RuntimeErrorEEEEE8grow_oneB19_
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer18floor_needs_adjust
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjmEE14reserve_rehashNCINvNtB8_3map11make_hasherjmNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECseWU5rFlE7Md_11weavepy_jit
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcDNtNtCs8Mbv00yxnRz_4core3any3AnyNtNtB1w_6marker4SyncNtB1Z_4SendEL_EE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zECs2D882usVKTN_10weavepy_vm
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjNtNtCseWU5rFlE7Md_11weavepy_jit7analyze7PairSrcNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertBS_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11do_sum_call
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjTINtNtCshxvaOLs88l5_5alloc4sync4WeakNtNtCs2D882usVKTN_10weavepy_vm6object9BuiltinFnEINtNtCs8Mbv00yxnRz_4core6option6OptionFG_RL0_SNtB1t_6ObjectEIB2e_INtNtB2i_6result6ResultB2X_NtNtB1v_5error12RuntimeErrorEEEEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtB1v_8fasthash13FxBuildHasherE0EB1v_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_upperINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins22sum_small_int_sequence
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjNtNtCseWU5rFlE7Md_11weavepy_jit7analyze7PairSrcEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBV_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_upper
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12ListLoopMetaE8grow_oneBQ_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins20sum_numeric_sequence
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19sum_exact_int_range
-__RNvMNtCs3fRkGY2FHAU_9hashbrown11rustc_entryINtNtB4_3map7HashMapmINtNtCshxvaOLs88l5_5alloc3vec3VecjENtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE11rustc_entryCseWU5rFlE7Md_11weavepy_jit
-__RNvNtCs2D882usVKTN_10weavepy_vm12unicode_case5upper
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_titleINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmINtNtCshxvaOLs88l5_5alloc3vec3VecjEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECseWU5rFlE7Md_11weavepy_jit
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_title
-__RNvNtCs2D882usVKTN_10weavepy_vm12unicode_case5title
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecThReNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters10filter_map9FilterMapINtNtNtB2s_5slice4iter4IterThB13_EENCNvNtNtB19_6stdlib15datetime_native7installs6_0EE9from_iterB19_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCshxvaOLs88l5_5alloc3vec3VecThReNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEB1y_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCshxvaOLs88l5_5alloc3vec3VecThReNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B2l_
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter21percent_mod_left_slot0B7_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit2ir11GlobalGuardE8grow_oneBQ_
-__RNCNCNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB9_8JitState12get_compileds0_0s3_0Bb_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEE8grow_oneCs2D882usVKTN_10weavepy_vm
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtNtNtBa_11collections4hash3map7HashMapReNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE4withNCNvNtB1G_8builtins12vm_intrinsic0INtNtCs8Mbv00yxnRz_4core6option6OptionB1C_EEB1G_
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapmTmNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeENtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertBU_
-__RNvXs7_NtCs2EXTltNQfP1_17cranelift_codegen4optsNtB5_11IsleContextNtNtB5_14generated_code7Context19iconst_sextend_etor
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc4sync3ArcNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectEE6borrowB7_
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtNtNtBe_11collections4hash3map7HashMapReNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEuE16get_or_init_slowNvNvNvNtB20_8builtins12vm_intrinsic5TABLE27___rust_std_internal_init_fnEB20_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtBG_6string6StringNtNtCseWU5rFlE7Md_11weavepy_jit2ir12CtorFieldSrcEEECs2D882usVKTN_10weavepy_vm
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmTmNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBX_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueNtB10_5BlockE20resize_for_index_mutB14_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectINtNtB7_4sync3ArcNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectEEE8grow_oneBR_
-__RNCNvNvNtCs2D882usVKTN_10weavepy_vm8builtins12vm_intrinsic5build0B7_
-__RNvMNtCs7fImLwyQvo9_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValuehE20resize_for_index_mutB14_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTReNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_BS_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBW_
-__RINvNtNtCs2EXTltNQfP1_17cranelift_codegen4opts14generated_code25constructor_ty_shift_maskNtB4_11IsleContextEB6_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_countINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins9str_count
-__RNvXs2_NtNtCs82bWklYMk3w_3std4hash6randomNtB5_13DefaultHasherNtNtCs8Mbv00yxnRz_4core4hash6Hasher5write
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5Valuej5_E8try_growBO_
-__RNvYFG_RL0_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtNtCs8Mbv00yxnRz_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins13b_set_tp_name
-__RINvNtNtCs2EXTltNQfP1_17cranelift_codegen4opts14generated_code20constructor_iconst_uNtB4_11IsleContextEB6_
-__RINvNtNtCs2EXTltNQfP1_17cranelift_codegen4opts14generated_code20constructor_iconst_sNtB4_11IsleContextEB6_
-__RINvXss_CsfNVduRRWYaN_8smallvecINtB6_8SmallVecANtNtNtCs2EXTltNQfP1_17cranelift_codegen4opts14generated_code26SkeletonInstSimplificationj5_EINtNtNtNtCs8Mbv00yxnRz_4core4iter6traits7collect6ExtendBJ_E6extendINtNtB2m_6option6OptionBJ_EEBP_
-__RINvNtNtCs2EXTltNQfP1_17cranelift_codegen4opts14generated_code24constructor_compose_icmpNtB4_11IsleContextEB6_
-__RNvMs1_NtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB5_9CtorState10note_store
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4math13make_checked10B7_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst3abi6ABIArgE8grow_oneBS_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4math6to_f64
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14coerce_f64_opt
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code24constructor_constant_f64INtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context25asimd_fp_mod_imm_from_u64
-__RNvMNtCs82bWklYMk3w_3std3f64d3sin
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215probe_list_lane
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code27constructor_fpu_move_fp_immINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215probe_dict_lane
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code24constructor_alu_rr_imm12INtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4math14freshen_result7freshen
-__RNvMNtCs82bWklYMk3w_3std3f64d3cos
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code23constructor_scalar_sizeINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4math7builtin0B7_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code19constructor_fpu_rrrINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib4math9math_sqrt
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native13plain_drop_ok
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze14boundary_stack
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context12fp_cond_code
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortjNvYjNtNtB8_3cmp10PartialOrd2ltECseWU5rFlE7Md_11weavepy_jit
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze9cmp_check
-__RNvMsd_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_11SlotStorage10get_hinted
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable9quicksort9quicksortjNvYjNtNtBa_3cmp10PartialOrd2ltECseWU5rFlE7Md_11weavepy_jit
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code23constructor_vec_dup_immINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code22constructor_int_to_fpuINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvXNtCs8Mbv00yxnRz_4core3anyNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native5StateNtB2_3Any7type_idBx_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native11verify_slow
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code40constructor_alu_rs_imm_logic_commutativeINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze12infer_append
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native6dt_add
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code19constructor_alu_rrrINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter12do_bool_call
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native9dt_fields
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapmTNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeBP_ENtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertBT_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native9td_fields
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code28constructor_alu_rr_imm_logicINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter20native_value_payload
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmTNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeBS_EEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBW_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13class_ns_load
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze10seed_entry
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native10valid_date
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code36constructor_imm12_from_negated_valueINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14class_ns_store
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context25extended_value_from_value
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native8dt_shift
-__RNvMs_NtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB4_4Plan9hidden_at
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter9stringify
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native7ymd2ord
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native7ord2ymd
-__RNvMs3_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5lowerINtB5_5LowerNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE28get_value_as_source_or_constB9_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze22resolve_dict_container
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native6new_dt
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native14instance_fixed
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code23constructor_lower_bmaskINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit2ir14MethodSpanMetaE8grow_oneBQ_
-__RNvMsb_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_11SlotStorage11from_layout
-__RNvXNtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapNtNtNtB2n_3str4iter5CharsNCNvB15_19str_subscript_slice0EE9from_iterB15_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit2ir14CalleeSpanMetaE8grow_oneBQ_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code22constructor_with_flagsINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs2_NtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB5_2SE8is_plain
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code31constructor_lower_icmp_into_regINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze22entry_unassigned_reads
-__RINvXs8_NtCshxvaOLs88l5_5alloc6stringNtB6_6StringINtNtNtNtCs8Mbv00yxnRz_4core4iter6traits7collect12FromIteratorBz_E9from_iterINtNtNtBS_8adapters3map3MapINtNtNtBU_5slice4iter4IterNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectENCNvB2U_19str_subscript_slices_0EEB2U_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native6dt_sub
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code32constructor_flags_and_cc_to_boolINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code35constructor_materialize_bool_resultINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs1_NtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB5_9CtorState10field_lane
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtCshxvaOLs88l5_5alloc5boxed3BoxDNtNtCs2EXTltNQfP1_17cranelift_codegen6timing8ProfilerEL_EEEB2j_
-__RNvMs1_NtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB5_9CtorState12stored_field
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native6new_td
-__RNCNvNtCs2D882usVKTN_10weavepy_vm12type_surface11unwrap_shim0B5_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code29constructor_put_in_reg_sext64INtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjThmmEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECseWU5rFlE7Md_11weavepy_jit
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code20constructor_do_shiftINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface11unwrap_shim14check_receiver
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native6td_mul
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context20imm_shift_from_imm64
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapmmNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCseWU5rFlE7Md_11weavepy_jit
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalEBO_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code28constructor_alu_rr_imm_shiftINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native6td_cmp
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze17dyn_attr_fallback
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code20constructor_alu_rrrrINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmmEE14reserve_rehashNCINvNtB8_3map11make_hashermmNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECseWU5rFlE7Md_11weavepy_jit
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins12dict_setitemINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1f_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins12dict_setitem
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code29constructor_put_in_reg_zext64INtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native7installs5_0B7_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native12date_weekday
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code18constructor_extendINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit2ir8OsrEntryE8grow_oneBQ_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native14date_toordinal
-__RNvYNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types22install_object_dunders16object_class_getINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir9sourceloc12RelSourceLocE8grow_oneBS_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native11date_fields
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCseWU5rFlE7Md_11weavepy_jit2ir13RangeLoopMetaE8grow_oneBQ_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types22install_object_dunders16object_class_get
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12weakref_real14proxy_referent
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214probe_obj_live
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6result6ResultNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtNtB11_5error12RuntimeErrorEEB11_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter25materialize_class_mapping
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native12dt_isoformat
-__RINvYNtNtCsgdPmlPE1JLX_18cranelift_frontend8frontend15FuncInstBuilderNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7builder11InstBuilder4jumpRINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB19_12instructions8BlockArgEECseWU5rFlE7Md_11weavepy_jit
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object9DictWatchEEB11_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native5tz_of
-__RNvXsZ_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt5Write10write_char
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native13format_offset
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native16dt_fromisoformat
-__RNvCs2D882usVKTN_10weavepy_vm14intersect_sets
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native9parse_hms
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21try_dict_from_mapping
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins6b_dictINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjNtNtCseWU5rFlE7Md_11weavepy_jit7analyze14CompHeaderPlanEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBV_
-__RNvMsK_NtCshxvaOLs88l5_5alloc4syncINtB5_4WeakNtNtCs2D882usVKTN_10weavepy_vm5types10PyInstanceE7upgradeBL_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins6b_dict
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native11dt_strftime
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze10cold_point
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native16strftime_numeric
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib15datetime_native8date_sub
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer5truth
-__RINvYNtNtCsgdPmlPE1JLX_18cranelift_frontend8frontend15FuncInstBuilderNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7builder11InstBuilder4brifRINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB19_12instructions8BlockArgEB2b_ECseWU5rFlE7Md_11weavepy_jit
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtCs2jIxScAqd7x_9regalloc25index5Blockj10_E21reserve_one_uncheckedCs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm5tier211MethodEntryEEB1c_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtCs2jIxScAqd7x_9regalloc25index5Blockj10_E8try_growCs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstEBN_
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer14pin_helper_sig
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit17enc_arith_rr_imml
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectINtNtBG_4sync3ArcNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectEEEEB1d_
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit14enc_arith_rrrr
-__RNvYNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types20install_type_dunders12type_delattrINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer10MachBranchj4_E21reserve_one_uncheckedBO_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectINtNtCshxvaOLs88l5_5alloc4sync3ArcNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectEEEBG_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types20install_type_dunders12type_delattr
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit7enc_bfm
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer14stage_elements
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons17queue_parked_drop
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit8enc_csel
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer15quad_helper_sig
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit12enc_inttofpu
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit14machreg_to_vec
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB1A_11Interpreter14run_activations_0E0zEB1A_
-__RNvMs2_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_8ListPoolNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE7reallocB13_
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjThjENtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCseWU5rFlE7Md_11weavepy_jit
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit8enc_fcmp
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB2o_11Interpreter14run_activations_0E0zE0E0B2o_
-__RINvMNtCs2D882usVKTN_10weavepy_vm8lazy_arcINtB3_7LazyArcINtNtB5_4sync7GilCellNtNtB5_6object8DictDataEE10initializeNCNvMs2_B5_NtB5_11Interpreter15leaf_store_attr0EB5_
-__RNvMse_NtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4argsNtB5_10ScalarSize5ftype
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjThjEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECseWU5rFlE7Md_11weavepy_jit
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit10enc_fpurrr
-__RNCNCNvNtCs2D882usVKTN_10weavepy_vm5tier212warm_compile00B7_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueTINtNtB4_6result6ResultNtNtCseWU5rFlE7Md_11weavepy_jit6engine13CompiledFrameNtNtB12_7analyze10JitVerdictEINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectINtNtB2n_4sync3ArcNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectEEEIB2j_NtNtB2U_5tier211MethodEntryEIB2j_NtNtB2n_6string6StringEEEB2U_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native12deque_append
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native10fast_parts
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native13bump_state_of
-__RNvMsd_NtCs2D882usVKTN_10weavepy_vm5typesNtB5_11SlotStorage14get_hinted_mut
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcDNtNtCs2EXTltNQfP1_17cranelift_codegen3isa9TargetIsaEL_E9drop_slowCslsAwG9FmMO9_13cranelift_jit
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14pure_leaf_eval
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native9maxlen_of
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch6414AArch64BackendEBH_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native13deque_popleft
-__RINvYNtNtCsgdPmlPE1JLX_18cranelift_frontend8frontend15FuncInstBuilderNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7builder11InstBuilder4brifRINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB19_12instructions8BlockArgERAB2J_j0_ECseWU5rFlE7Md_11weavepy_jit
-__RNvXNvMs2_Cs2D882usVKTN_10weavepy_vmNtB8_11Interpreter14pure_leaf_evalNtB2_5OwnedNtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4drop
-__RNvYNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7builder14ReplaceBuilderNtB4_11InstBuilder4iaddB8_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native7head_of
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCshfBCtCVroC4_14target_lexicon6triple6TripleECs2EXTltNQfP1_17cranelift_codegen
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native12popleft_fast
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc4sync4WeakDNtNtCs2EXTltNQfP1_17cranelift_codegen3isa9TargetIsaEL_RNtNtBG_5alloc6GlobalEECslsAwG9FmMO9_13cranelift_jit
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins10b_callableINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_5boxed3BoxDG_INtNtNtB4_3ops8function2FnTRL0_eEEp6OutputINtNtB4_6option6OptionPhENtNtB4_6marker4SendEL_EEECs2D882usVKTN_10weavepy_vm
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native13deque_getitem
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins18object_is_callable
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins4b_idINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxDG_INtNtNtB4_3ops8function2FnTRL0_eEEp6OutputINtNtB4_6option6OptionPhENtNtB4_6marker4SendEL_EECs2D882usVKTN_10weavepy_vm
-__RNvMs0_NtCsgSMFPVT6ofn_17cranelift_bforest4nodeINtB5_8NodeDataINtNtB7_3map8MapTypesNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities4InstNtB1l_5BlockEE15try_leaf_insertB1p_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins19coerce_index_object
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native8receiver
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native10DequeStateEBH_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins15object_identity
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native10deque_bool
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCslsAwG9FmMO9_13cranelift_jit6memory6system20SystemMemoryProviderEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCslsAwG9FmMO9_13cranelift_jit6memory6system6MemoryEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueNtNtBW_20remove_constant_phis13AbstractValueEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1L_NtCsfoFpBBKvNFT_10rustc_hash13FxBuildHasherE0EBW_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze22resolve_list_container
-__RNvMs1_NtCs6MP6LIxl4z2_16weavepy_compiler8bytecodeNtB5_6OpCode4name
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCslsAwG9FmMO9_13cranelift_jit6memory6system6PtrLenEEB1e_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs8MhVFxlpMWQ_16cranelift_module6module18ModuleDeclarationsECs2D882usVKTN_10weavepy_vm
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native20deque_iterator_index
-__RNvMNtCs3fRkGY2FHAU_9hashbrown11rustc_entryINtNtB4_3map7HashMapmmNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE11rustc_entryCseWU5rFlE7Md_11weavepy_jit
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins16coerce_index_i64
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins20try_coerce_index_i64
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native19deque_iterator_next
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native10deque_next
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtBG_13tuple_storage12TupleStorageEEBG_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native4head
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm6object14SlotDescriptorE9drop_slowBK_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter9op_binary
-__RNvMs_NtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB4_4Plan18comp_lane_for_body
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native16deque_appendleft
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectE9drop_slowBK_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCshxvaOLs88l5_5alloc3vec6splice6SpliceINtNtNtNtB4_4iter7sources8repeat_n7RepeatNNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB24_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB1o_4sync4WeakNtNtBG_5types10TypeObjectEEEEBG_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11do_len_call
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs7fImLwyQvo9_16cranelift_entity3map12SecondaryMapNtNtCs8MhVFxlpMWQ_16cranelift_module6module6DataIdINtNtB4_6option6OptionNtNtCslsAwG9FmMO9_13cranelift_jit13compiled_blob12CompiledBlobEEECs2D882usVKTN_10weavepy_vm
-__RNvMs3_NtCs7fImLwyQvo9_16cranelift_entity4listINtB5_10EntityListNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities5ValueE4growB16_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib18collections_native9deque_len
-__RNCNCNvNtCs2D882usVKTN_10weavepy_vm5tier212warm_compile0s0_0B7_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter16core_pure_method
-__RNvCs2D882usVKTN_10weavepy_vm17coerce_len_result
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtNtCs82bWklYMk3w_3std4sync9once_lock8OnceLockINtNtCshxvaOLs88l5_5alloc4sync3ArcDNtNtB4_3any3AnyNtNtB4_6marker4SyncNtB2e_4SendEL_EEECs2D882usVKTN_10weavepy_vm
-__RNCNCNvNtCs2D882usVKTN_10weavepy_vm5tier212warm_compile0s1_0B7_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2EXTltNQfP1_17cranelift_codegen13loop_analysis8LoopDataE8grow_oneBQ_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2EXTltNQfP1_17cranelift_codegen7context7ContextECs2D882usVKTN_10weavepy_vm
-__RNCNCNvNtCs2D882usVKTN_10weavepy_vm5tier212warm_compile0s7_0B7_
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapmNtNtCseWU5rFlE7Md_11weavepy_jit2ir13CompSavedMetaNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertBS_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmNtNtCseWU5rFlE7Md_11weavepy_jit2ir13CompSavedMetaEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBV_
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCseWU5rFlE7Md_11weavepy_jit7analyze2SEEINtB4_18SpecFromIterNestedB13_INtNtB6_5drain5DrainB13_EE9from_iterB17_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftNtNtCseWU5rFlE7Md_11weavepy_jit2ir8OsrEntryNCINvMB8_SB1m_20sort_unstable_by_keymNCNvNtB1q_7analyze12analyze_onces0_0E0EB1q_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze14escape_verdict
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8constant12ConstantPoolECs2D882usVKTN_10weavepy_vm
-__RNvMsz_NtNtNtCshxvaOLs88l5_5alloc11collections5btree3mapINtB5_8IntoIterNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8constant12ConstantDataNtNtB1b_8entities8ConstantE10dying_nextCs2D882usVKTN_10weavepy_vm
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code27constructor_aarch64_uload32INtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvNtCs2jIxScAqd7x_9regalloc27domtree10merge_sets
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs7fImLwyQvo9_16cranelift_entity7primary10PrimaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities9ImmediateNtNtB1A_8constant12ConstantDataEECs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures13BlockparamOutE8grow_oneCs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs7fImLwyQvo9_16cranelift_entity7primary10PrimaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities9JumpTableNtNtB1A_9jumptable13JumpTableDataEECs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12BlockparamInE8grow_oneCs2EXTltNQfP1_17cranelift_codegen
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12BlockparamInNCINvMB8_SB1m_20sort_unstable_by_keyoNCNvMs0_NtB1q_10liverangesINtB1o_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3I_3isa7aarch645lower4isle14generated_code5MInstEE16build_liveranges0E0EB3I_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs7fImLwyQvo9_16cranelift_entity7primary10PrimaryMapNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8entities14ExceptionTableNtNtB1A_15exception_table18ExceptionTableDataEECs2D882usVKTN_10weavepy_vm
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures13BlockparamOutNCINvMB8_SB1m_20sort_unstable_by_keyoNCNvMs0_NtB1q_10liverangesINtB1o_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3J_3isa7aarch645lower4isle14generated_code5MInstEE16build_liverangess_0E0EB3J_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir6layout6LayoutECs2D882usVKTN_10weavepy_vm
-__RINvXs_NtCs2EXTltNQfP1_17cranelift_codegen7ctxhashNtB5_7NullCtxINtB5_7CtxHashTNtNtNtB7_2ir5types4TypeNtNtB1i_12instructions15InstructionDataEE8ctx_hashNtCsfoFpBBKvNFT_10rustc_hash8FxHasherEB7_
-__RNvXsc_NtNtCs8Mbv00yxnRz_4core5slice5indexTINtNtNtB9_3ops5range5BoundjEBF_EINtB5_10SliceIndexSNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures18LiveRangeListEntryE5indexCs2EXTltNQfP1_17cranelift_codegen
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter14pure_leaf_evals_0B7_
-__RNvYNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir7builder14ReplaceBuilderNtB4_11InstBuilder4bxorB8_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir10debug_tags9DebugTagsECs2D882usVKTN_10weavepy_vm
-__RNvXs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcodeINtB5_5VCodeNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstENtCs2jIxScAqd7x_9regalloc28Function14spillslot_sizeB9_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2EXTltNQfP1_17cranelift_codegen2ir8function18FunctionParametersECs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures13SpillSlotDataE8grow_oneCs2EXTltNQfP1_17cranelift_codegen
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion5spillINtNtB4_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1e_3isa7aarch645lower4isle14generated_code5MInstEE30allocate_spillset_to_spillslotB1e_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2EXTltNQfP1_17cranelift_codegen9flowgraph16ControlFlowGraphECs2D882usVKTN_10weavepy_vm
-__RNCNCNvNtCs2D882usVKTN_10weavepy_vm5tier212warm_compile0sa_0B7_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier211guards_hold
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context18u64_into_imm_logic
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier28take_u64
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code24constructor_lower_selectINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code16constructor_cselINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2EXTltNQfP1_17cranelift_codegen14dominator_tree13DominatorTreeECs2D882usVKTN_10weavepy_vm
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellNtNtCs2D882usVKTN_10weavepy_vm5tier27JitBufsEuE16get_or_init_slowNvNvB1R_8JIT_BUFS27___rust_std_internal_init_fnEB1T_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier28take_u32
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2D882usVKTN_10weavepy_vm5tier23PinE8grow_oneBQ_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code30constructor_put_nonzero_in_regINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2EXTltNQfP1_17cranelift_codegen13loop_analysis12LoopAnalysisECs2D882usVKTN_10weavepy_vm
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs8Mbv00yxnRz_4core4cell7RefCellNtNtCs2D882usVKTN_10weavepy_vm5tier28JitStateEE4withNCNvB1x_19current_compile_gen0yEB1z_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214wpjit_attr_get
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier211unpack_pins
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCs2EXTltNQfP1_17cranelift_codegen8machinst16CompiledCodeBaseNtNtB10_6buffer5FinalEEECs2D882usVKTN_10weavepy_vm
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier221drain_activation_pins
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCsgdPmlPE1JLX_18cranelift_frontend8frontend22FunctionBuilderContextECs2D882usVKTN_10weavepy_vm
-__RNvXs5_NtNtCshxvaOLs88l5_5alloc3vec5drainINtB5_5DrainNtNtCs2D882usVKTN_10weavepy_vm5tier23PinENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropBT_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecyEE8grow_oneCs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecmEE8grow_oneCs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5tier27CallCtxEBF_
-__RNvXs0_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch643abiNtB5_18AArch64MachineDepsNtNtNtBb_8machinst3abi14ABIMachineSpec17gen_sp_reg_adjust
-__RINvXss_CsfNVduRRWYaN_8smallvecINtB6_8SmallVecANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj10_EINtNtNtNtCs8Mbv00yxnRz_4core4iter6traits7collect6ExtendBJ_E6extendIBu_ABJ_j4_EEBV_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code5MInstj10_E8try_growBU_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm5tier23PinEEB1c_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecATmNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa6unwind10UnwindInstEj8_E21reserve_one_uncheckedBQ_
-__RNvYNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13itertools_mod10count_coreINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNvMs0_NtNtCs2jIxScAqd7x_9regalloc23ion11requirementINtNtB7_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1o_3isa7aarch645lower4isle14generated_code5MInstEE25merge_bundle_requirementsB1o_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13itertools_mod10count_core
-__RNvMs_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644instNtNtNtNtB6_5lower4isle14generated_code5MInst9gen_store
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real13allocate_lock
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real16make_lock_object
-__RNvMNtNtCs2jIxScAqd7x_9regalloc23ion7processINtNtB4_15data_structures3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB1g_3isa7aarch645lower4isle14generated_code5MInstEE26split_into_minimal_bundlesB1g_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13functools_mod5build
-__RNvXsq_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RNCNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13functools_mod8lru_type00B9_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VechEECs2D882usVKTN_10weavepy_vm
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11do_map_call
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real16new_rlock_object
-__RNvMs_NtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644instNtNtNtNtB6_5lower4isle14generated_code5MInst8gen_load
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter18make_seqtools_iter
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtNtNtBa_11collections4hash3map7HashMapReNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1Q_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter11do_str_call
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter12load_special
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys10sys_intern
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter12load_special0B7_
-__RINvMs10_NtNtNtCshxvaOLs88l5_5alloc11collections5btree4nodeINtB7_16BalancingContextNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12LiveRangeKeyNtB1l_14LiveRangeIndexE25merge_tracking_child_edgeNtNtBd_5alloc6GlobalECs2EXTltNQfP1_17cranelift_codegen
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real26instance_dunder_trampoline0B7_
-__RNvMs10_NtNtNtCshxvaOLs88l5_5alloc11collections5btree4nodeINtB6_16BalancingContextNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12LiveRangeKeyNtB1k_14LiveRangeIndexE15bulk_steal_leftCs2EXTltNQfP1_17cranelift_codegen
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real16make_lock_objects1_0B7_
-__RNvMsA_NtCs2D882usVKTN_10weavepy_vm4syncNtB5_8RealLock11try_acquire
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins16str_isidentifierINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214entry_local_ok
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier220decompose_live_loops
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins16str_isidentifier
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real16make_lock_objects2_0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real17thread_handle_new
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins12obj_containsINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code29constructor_aarch64_fpuload64INtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real25make_thread_handle_object
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtNtNtB7_11collections4hash3map7HashMapyINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real17ThreadHandleStateEEEE10initializeNCINvB2_11get_or_initNCNvB3C_15handle_registry0E0zEB3G_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214enter_compiled
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12BlockparamInNCINvMB6_SBT_20sort_unstable_by_keyoNCNvMs0_NtBX_10liverangesINtBV_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3c_3isa7aarch645lower4isle14generated_code5MInstEE16build_liveranges0E0EB3c_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable9quicksort9quicksortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12BlockparamInNCINvMB8_SB17_20sort_unstable_by_keyoNCNvMs0_NtB1b_10liverangesINtB19_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3t_3isa7aarch645lower4isle14generated_code5MInstEE16build_liveranges0E0EB3t_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTyINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real17ThreadHandleStateEEE14reserve_rehashNCINvNtB8_3map11make_hasheryBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1v_
-__RNvMs10_NtNtNtCshxvaOLs88l5_5alloc11collections5btree4nodeINtB6_16BalancingContextNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12LiveRangeKeyNtB1k_14LiveRangeIndexE16bulk_steal_rightCs2EXTltNQfP1_17cranelift_codegen
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12InsertedMoveNCINvMB6_SBT_20sort_unstable_by_keyyNCNvMNtBX_5movesINtBV_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB33_3isa7aarch645lower4isle14generated_code5MInstEE22resolve_inserted_moves0E0EB33_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared5pivot11median3_recNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12BlockparamInNCINvMB8_SB14_20sort_unstable_by_keyoNCNvMs0_NtB18_10liverangesINtB16_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3q_3isa7aarch645lower4isle14generated_code5MInstEE16build_liveranges0E0EB3q_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable9quicksort9quicksortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12InsertedMoveNCINvMB8_SB17_20sort_unstable_by_keyyNCNvMNtB1b_5movesINtB19_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3k_3isa7aarch645lower4isle14generated_code5MInstEE22resolve_inserted_moves0E0EB3k_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins24bytearray_receiver_guard
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object20ByteArrayExportGuardEEB11_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real15new_lock_object
-__RNvNtCs2D882usVKTN_10weavepy_vm6object19current_interp_hash
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17merge_exec_kwargs
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter12do_eval_call
-__RNvCs2D882usVKTN_10weavepy_vm26check_compile_source_nulls
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures13BlockparamOutNCINvMB6_SBT_20sort_unstable_by_keyoNCNvMs0_NtBX_10liverangesINtBV_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3d_3isa7aarch645lower4isle14generated_code5MInstEE16build_liverangess_0E0EB3d_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable9quicksort9quicksortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures13BlockparamOutNCINvMB8_SB17_20sort_unstable_by_keyoNCNvMs0_NtB1b_10liverangesINtB19_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3u_3isa7aarch645lower4isle14generated_code5MInstEE16build_liverangess_0E0EB3u_
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcNtNtCs2D882usVKTN_10weavepy_vm5tier29ArtifactsE9drop_slowBH_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared5pivot11median3_recNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures13BlockparamOutNCINvMB8_SB14_20sort_unstable_by_keyoNCNvMs0_NtB18_10liverangesINtB16_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3r_3isa7aarch645lower4isle14generated_code5MInstEE16build_liverangess_0E0EB3r_
-__RNvCs1LCFj3BjrAK_14weavepy_parser19parse_with_warnings
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5tier29ArtifactsEBF_
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser10parse_eval
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcNtNtCseWU5rFlE7Md_11weavepy_jit6engine13CompiledFrameE9drop_slowCs2D882usVKTN_10weavepy_vm
-__RNvNtCs1LCFj3BjrAK_14weavepy_parser6parser21parse_eval_with_flufl
-__RNCINvMsZ_NtCs2D882usVKTN_10weavepy_vm6objectNtB8_9BuiltinFn11with_kwargsNCNvNtNtBa_6stdlib11thread_real16make_lock_object0Es_0Ba_
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser17parse_eval_module
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real16make_lock_object0B7_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit6engine13CompiledFrameECs2D882usVKTN_10weavepy_vm
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real20resolve_acquire_args
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser18parse_eval_element
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier217wpjit_list_append
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214wpjit_list_len
-__RNvMNtCs1LCFj3BjrAK_14weavepy_parser6parserNtB2_6Parser12parse_lambda
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real13parse_timeout
-__RNvCs1LCFj3BjrAK_14weavepy_parser21eval_backslash_at_eof
-__RNvNvCs2D882usVKTN_10weavepy_vm32collect_compiler_syntax_warnings10visit_body
-__RNvNvCs2D882usVKTN_10weavepy_vm32collect_compiler_syntax_warnings10visit_expr
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real21get_main_thread_ident
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real18make_thread_handle
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable14driftsort_mainTNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures11PosWithPrioNtB16_4EditENCINvMNtCshxvaOLs88l5_5alloc5sliceSBZ_11sort_by_keyyNCNvMsp_B12_NtB12_5Edits4sort0E0INtNtB2m_3vec3VecBZ_EECs2EXTltNQfP1_17cranelift_codegen
-__RNvNvCs2D882usVKTN_10weavepy_vm32collect_compiler_syntax_warnings10visit_args
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable5drift4sortTNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures11PosWithPrioNtB13_4EditENCINvMNtCshxvaOLs88l5_5alloc5sliceSBW_11sort_by_keyyNCNvMsp_BZ_NtBZ_5Edits4sort0E0ECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCsftQRLRmRWkP_13weavepy_lexer5token13EscapeWarningEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real25make_thread_handle_object0EBJ_
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable14driftsort_mainINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer10MachSrcLocNtB12_7StencilENCINvMNtCshxvaOLs88l5_5alloc5sliceSBZ_11sort_by_keymNCNvMs4_B12_INtB12_10MachBufferNtNtNtNtNtNtB16_3isa7aarch645lower4isle14generated_code5MInstE6finishs0_0E0INtNtB2s_3vec3VecBZ_EEB16_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable5drift4sortINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer10MachSrcLocNtBZ_7StencilENCINvMNtCshxvaOLs88l5_5alloc5sliceSBW_11sort_by_keymNCNvMs4_BZ_INtBZ_10MachBufferNtNtNtNtNtNtB13_3isa7aarch645lower4isle14generated_code5MInstE6finishs0_0E0EB13_
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures18LiveRangeListEntryNCINvMB6_SBT_20sort_unstable_by_keyNtBZ_9ProgPointNCNvMNtBX_5movesINtBV_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3n_3isa7aarch645lower4isle14generated_code5MInstEE34apply_allocations_and_insert_moves0E0EB3n_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter25exec_eval_compile_options
-__RNvCs6MP6LIxl4z2_16weavepy_compiler25compile_eval_with_options
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real13get_native_id
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real17make_rlock_objects6_0B7_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst10blockorder12LoweredBlockj40_E8try_growBO_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real17make_rlock_objects7_0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10os_process19register_at_fork_kw
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit2ir12AttrSiteMetaEECs2D882usVKTN_10weavepy_vm
-__RNvXs1k_NtCs2D882usVKTN_10weavepy_vm6objectNtB6_6ObjectNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTmNtNtCs2D882usVKTN_10weavepy_vm5tier29SynthSeedEEEB1e_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBG_6string6StringEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEECs6MP6LIxl4z2_16weavepy_compiler
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5tier213CompiledEntryEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast7KeywordEECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecAINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer14MachLabelFixupNtNtNtNtNtNtBP_3isa7aarch645lower4isle14generated_code5MInstEj10_E21reserve_one_uncheckedBP_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze16infer_store_attr
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionTmINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB14_6string6StringENtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeIBC_TB1w_mEEbEEEB1X_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecAhj400_E8try_growCs2EXTltNQfP1_17cranelift_codegen
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared5pivot11median3_recNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures12InsertedMoveNCINvMB8_SB14_20sort_unstable_by_keyyNCNvMNtB18_5movesINtB16_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3h_3isa7aarch645lower4isle14generated_code5MInstEE22resolve_inserted_moves0E0EB3h_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast8StmtKindECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast9ArgumentsECs2D882usVKTN_10weavepy_vm
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcNtNtCs2D882usVKTN_10weavepy_vm5tier213GuardSnapshotE9drop_slowBH_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter14lazy_iter_next
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast3ArgEECs2D882usVKTN_10weavepy_vm
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtBG_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB1y_
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcINtNtB7_3vec3VecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectINtNtB7_4sync3ArcNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectEEEE9drop_slowBY_
-__RNvXNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isleINtNtNtBa_8machinst4isle11IsleContextNtNtB2_14generated_code5MInstNtB6_14AArch64BackendENtB1H_7Context26test_and_compare_bit_const
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs1LCFj3BjrAK_14weavepy_parser3ast3ArgECs2D882usVKTN_10weavepy_vm
-__RINvNtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_code15constructor_tbzINtNtNtBc_8machinst4isle11IsleContextNtB2_5MInstNtB8_14AArch64BackendEEBc_
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcINtNtB7_3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE9drop_slowBX_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEEECs2D882usVKTN_10weavepy_vm
-__RNvXs9_NtNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch645lower4isle14generated_codeNtB5_18SideEffectNoResultNtNtCs8Mbv00yxnRz_4core5clone5Clone5clone
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBW_6string6StringEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECseWU5rFlE7Md_11weavepy_jit
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcINtNtB7_3vec3VecNtNtCs2D882usVKTN_10weavepy_vm5tier29AttrGuardEE9drop_slowBX_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214wpjit_attr_set
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm5tier29AttrGuardEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEECs2D882usVKTN_10weavepy_vm
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRTmNtNtCshxvaOLs88l5_5alloc6string6StringEECseWU5rFlE7Md_11weavepy_jit
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5tier29AttrGuardEBF_
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcINtNtB7_3vec3VecNtNtCs2D882usVKTN_10weavepy_vm5tier211MethodEntryEE9drop_slowBX_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTTmNtNtCshxvaOLs88l5_5alloc6string6StringENtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1v_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1z_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtNtCs1LCFj3BjrAK_14weavepy_parser3ast4ExprEECs2D882usVKTN_10weavepy_vm
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtCs2jIxScAqd7x_9regalloc210Allocationj4_E21reserve_one_uncheckedBK_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins16mappingproxy_getINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs1LCFj3BjrAK_14weavepy_parser3ast7KeywordEECs2D882usVKTN_10weavepy_vm
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcINtNtB7_3vec3VecTNtNtB7_6string6StringBU_NtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEE9drop_slowB1m_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins16mappingproxy_get
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins21mappingproxy_delegate
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtBG_6string6StringB19_NtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB1C_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins17mappingproxy_args
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalEBO_
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcINtNtB7_3vec3VecINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCs2D882usVKTN_10weavepy_vm5tier212NativeCalleeEEE9drop_slowB1z_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real9get_ident
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm5tier212NativeCalleeEEEB1y_
-__RNvYNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types20install_type_dunders12type_setattrINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter30winner_aware_dynamic_type_call
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types20install_type_dunders12type_setattr
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins7set_popINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNCINvMsZ_NtCs2D882usVKTN_10weavepy_vm6objectNtB8_9BuiltinFn11with_kwargsNvNtNtBa_6stdlib11thread_real21start_joinable_threadEs_0Ba_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins7set_pop
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEBG_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys16build_with_states_0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real19spawn_python_worker
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyuNtNtBS_8fasthash13FxBuildHasherE17shift_remove_fullBO_EBS_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys22sys_getframemodulename
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm15thread_registry14ThreadRegistryE10initializeNCINvB2_11get_or_initNvMs0_BV_BT_3newE0zEBX_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm13vm_singletons13ThreadHandlesEBF_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs2D882usVKTN_10weavepy_vm15thread_registry14ThreadRegistryE10initializeNCINvB1a_11get_or_initNvMs0_B1I_B1G_3newE0zE0E0B1K_
-__RNvYNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13functools_mod12partial_callINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1R_EEE4callB8_
-__RNvMNtCs2D882usVKTN_10weavepy_vm15thread_registryNtB2_11ThreadEntry3new
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib13functools_mod12partial_call
-__RINvXNvMNtCshxvaOLs88l5_5alloc5sliceSp9to_vec_inNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalEBO_
-__RNvMs0_NtCs2D882usVKTN_10weavepy_vm15thread_registryNtB5_14ThreadRegistry8register
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE8grow_oneB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtBG_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB1y_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtBG_13tuple_storage12TupleStorageEEBG_
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures18LiveRangeListEntryNCINvMB6_SBT_20sort_unstable_by_keyNtBZ_9ProgPointNCNvMNtBX_5mergeINtBV_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3n_3isa7aarch645lower4isle14generated_code5MInstEE13merge_bundless1_0E0EB3n_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real26lock_acquire_interruptible
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10struct_mod5build
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable9quicksort9quicksortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures18LiveRangeListEntryNCINvMB8_SB17_20sort_unstable_by_keyNtB1d_9ProgPointNCNvMNtB1b_5mergeINtB19_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3F_3isa7aarch645lower4isle14generated_code5MInstEE13merge_bundless1_0E0EB3F_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types20install_type_dunders12type_doc_get
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortNtNvMNtNtCs2jIxScAqd7x_9regalloc23ion5movesINtNtB10_15data_structures3EnvpE34apply_allocations_and_insert_moves14InterBlockDestNCINvMB6_SBT_20sort_unstable_by_keyyNvMs_BV_BT_3keyE0ECs2EXTltNQfP1_17cranelift_codegen
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable9quicksort9quicksortNtNvMNtNtCs2jIxScAqd7x_9regalloc23ion5movesINtNtB1e_15data_structures3EnvpE34apply_allocations_and_insert_moves14InterBlockDestNCINvMB8_SB17_20sort_unstable_by_keyyNvMs_B19_B17_3keyE0ECs2EXTltNQfP1_17cranelift_codegen
-__RNvYNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io15io_abc_registerINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib2io15io_abc_register
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real16make_lock_objects_0B7_
-__RNvNtCs2D882usVKTN_10weavepy_vm12type_surface20obj_contains_builtin
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10codecs_mod5build
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm3gil8GilGuardEEB1c_
-__RNvCs2D882usVKTN_10weavepy_vm25detect_yield_from_subiter
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyNtNtCs2D882usVKTN_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_13key_cmp_scopeINtNtCs8Mbv00yxnRz_4core6option6OptionNtBW_6ObjectENCNvMs2_BY_NtBY_11Interpreter4stepsC_0E0INtNtB2n_6result6ResultB2i_NtNtBY_5error12RuntimeErrorEEBY_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm3gil8GilGuardEBF_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface22install_object_compare6obj_eq
-__RNvMsA_NtCs2D882usVKTN_10weavepy_vm4syncNtB5_8RealLock15acquire_timeout
-__RNvMs_NtCseaAvxSPddma_11parking_lot7condvarNtB4_7Condvar19wait_until_internal
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageNtNtCs6I8f3QNB0wx_16parking_lot_core11parking_lot10ThreadDatauE16get_or_init_slowNvNvNvB1g_16with_thread_data11THREAD_DATA27___rust_std_internal_init_fnECseaAvxSPddma_11parking_lot
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215callee_ret_lane
-__RNvMs1_NtCs6I8f3QNB0wx_16parking_lot_core11parking_lotNtB5_10ThreadData3new
-__RNvMNtCs6I8f3QNB0wx_16parking_lot_core11parking_lotNtB2_9HashTable3new
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE10defer_trapB9_
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyNtNtCs2D882usVKTN_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_13key_cmp_scopeINtNtCs8Mbv00yxnRz_4core6option6OptionNtBW_6ObjectENCNvMs2_BY_NtBY_11Interpreter11instantiates_0E0INtNtB2n_6result6ResultB2i_NtNtBY_5error12RuntimeErrorEEBY_
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE9get_labelB9_
-__RNvMs_NtCs2D882usVKTN_10weavepy_vm6importNtB4_11ModuleCache6remove
-__RINvMs0_NtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapPNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectTINtNtB1j_6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEbINtNtCshxvaOLs88l5_5alloc4sync3ArcB2w_EEEEuE16get_or_init_slowNvNvNtCs2D882usVKTN_10weavepy_vm5tier214RET_LANE_CACHE27___rust_std_internal_init_fnEB5v_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15analyze_for_ret
-__RNSNvYNCINvNtNtCs82bWklYMk3w_3std6thread9lifecycle15spawn_uncheckedNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real19spawn_python_workers1_0uEs_0INtNtNtCs8Mbv00yxnRz_4core3ops8function6FnOnceuE9call_once6vtableB1b_
-__RINvNtNtCs82bWklYMk3w_3std3sys9backtrace28___rust_begin_short_backtraceNCNCNCINvNtNtB6_6thread9lifecycle15spawn_uncheckedNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real19spawn_python_workers1_0uEs_000uEB23_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_7PyFrame15locals_snapshot
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215callee_ret_infos0_0B5_
-__RINvNtNtCs82bWklYMk3w_3std3sys9backtrace28___rust_begin_short_backtraceNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real19spawn_python_workers1_0uEB1f_
-__RNCNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15analyze_for_rets5_0B5_
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_7PyFrame22has_live_hidden_locals
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer14MachLabelFixupNtNtNtNtNtNtBT_3isa7aarch645lower4isle14generated_code5MInstEE8grow_oneBT_
-__RNvMs9_NtNtCshxvaOLs88l5_5alloc11collections11binary_heapINtB5_10BinaryHeapINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer14MachLabelFixupNtNtNtNtNtNtB1i_3isa7aarch645lower4isle14generated_code5MInstEE7sift_upB1i_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit7analyze9PathArenaEBF_
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons24install_worker_thread_id
-__RNvMs2_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_7PyFrame14compute_locals
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE8add_trapB9_
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable14driftsort_mainNtNtCshxvaOLs88l5_5alloc6string6StringNvYBZ_NtNtB8_3cmp10PartialOrd2ltINtNtB13_3vec3VecBZ_EECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMs4_NtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6bufferINtB5_10MachBufferNtNtNtNtNtNtB9_3isa7aarch645lower4isle14generated_code5MInstE8put_dataB9_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable5drift4sortNtNtCshxvaOLs88l5_5alloc6string6StringNvYBW_NtNtBa_3cmp10PartialOrd2ltECs6MP6LIxl4z2_16weavepy_compiler
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTyyEE14reserve_rehashNCINvNtB8_3map11make_hasheryyNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6stable9quicksort9quicksortNtNtCshxvaOLs88l5_5alloc6string6StringNvYB15_NtNtBa_3cmp10PartialOrd2ltECs6MP6LIxl4z2_16weavepy_compiler
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons26install_seed_builtin_types
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared5pivot11median3_recNtNtCshxvaOLs88l5_5alloc6string6StringNvYB14_NtNtBa_3cmp10PartialOrd2ltECs6MP6LIxl4z2_16weavepy_compiler
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types14install_shared
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort31small_sort_general_with_scratchNtNtCshxvaOLs88l5_5alloc6string6StringNvYB1s_NtNtBa_3cmp10PartialOrd2ltECs6MP6LIxl4z2_16weavepy_compiler
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTPNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectTINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEbINtNtCshxvaOLs88l5_5alloc4sync3ArcBR_EEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1B_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort12sort4_stableNtNtCshxvaOLs88l5_5alloc6string6StringNvYB19_NtNtBa_3cmp10PartialOrd2ltECs6MP6LIxl4z2_16weavepy_compiler
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecONtB7_11InterpreterEE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB7_13vm_singletons13ThreadHandlesEE21try_borrow_mut_lockedB7_
-__RINvNtNtCshxvaOLs88l5_5alloc3vec16in_place_collect18from_iter_in_placeINtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtB4_9into_iter8IntoIterNtNtB6_6string6StringEINvMs1j_NtCs2D882usVKTN_10weavepy_vm6objectNtB2P_6Object8from_strB2l_EEB3o_EB2R_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtNtNtCs82bWklYMk3w_3std11collections4hash3set7HashSetNtNtB7_12shared_value9SharedStrEE21try_borrow_mut_lockedB7_
-__RNCNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15analyze_for_rets_0B5_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11str_isupperINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNCNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15analyze_for_rets0_0B5_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBV_4sync3ArcNtNtB7_6object10FrameShellEEE21try_borrow_mut_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11str_isupper
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_5types10TypeObjectEE17try_borrow_lockedB7_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringNtNtCseWU5rFlE7Md_11weavepy_jit2ir12CtorFieldSrcEE8grow_oneCs2D882usVKTN_10weavepy_vm
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBV_4sync3ArcNtNtB7_5types10TypeObjectEEE17try_borrow_lockedB7_
-__RNCNCNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB9_8JitState12get_compileds0_0s_0Bb_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object8DictDataE17try_borrow_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecTjINtNtBV_4sync3ArcNtNtB7_5types10PyInstanceEEEE17try_borrow_lockedB7_
-__RNCNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15analyze_for_ret0B5_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBV_4sync3ArcNtNtB7_8gc_trace13TrackedHandleEEE21try_borrow_mut_lockedB7_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215callee_ret_infos1_0B5_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueTmINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBI_6string6StringEB1a_EECs2D882usVKTN_10weavepy_vm
-__RNCNCNvNtCs2D882usVKTN_10weavepy_vm5tier212warm_compile0s4_0B7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_16weakref_registry13RegistryInnerE17try_borrow_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier218probe_method_entry
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtB7_6object6ObjectEE17try_borrow_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215method_ret_info
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object8DictDataE21try_borrow_mut_lockedB7_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215method_ret_infos_0B5_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB7_6object6ObjectEE17try_borrow_lockedB7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel5build
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB7_6object6ObjectEE21try_borrow_mut_lockedB7_
-__RNvYNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel10make_loadsINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215method_ret_infos8_0B5_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel10make_loads
-__RINvYNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateNtNtCs8Mbv00yxnRz_4core4hash11BuildHasher8hash_oneRTmINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtB1M_6string6StringEB2e_EECs2D882usVKTN_10weavepy_vm
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_5types11SlotStorageE21try_borrow_mut_lockedB7_
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortNtNtNtCs2jIxScAqd7x_9regalloc23ion15data_structures18LiveRangeListEntryNCINvMB6_SBT_20sort_unstable_by_keyNtBZ_9ProgPointNCNvMNtBX_5spillINtBV_3EnvINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst5vcode5VCodeNtNtNtNtNtNtB3n_3isa7aarch645lower4isle14generated_code5MInstEE39try_allocating_regs_for_spilled_bundles0E0EB3n_
-__RNvMNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB2_13FunctionGuard3new
-__RNvMNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB2_13FunctionGuard5holds
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTTmINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtBX_6string6StringEB1p_EmEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_mNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtCs2D882usVKTN_10weavepy_vm5tier211MethodEntryE8grow_oneBQ_
-__RNvMs_NtCseaAvxSPddma_11parking_lot7condvarNtB4_7Condvar15notify_one_slow
-__RNvNtCs6I8f3QNB0wx_16parking_lot_core11parking_lot16lock_bucket_pair
-__RINvMNtCs8Mbv00yxnRz_4core6optionINtB3_6OptionNtNtCseWU5rFlE7Md_11weavepy_jit7analyze16MethodResolutionE7or_elseNCNvBK_13step_abstracts3_0EBM_
-__RNvMs1_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_11ImportGuard3new
-__RNvMs1_NtCseaAvxSPddma_11parking_lot9raw_mutexNtB5_8RawMutex9lock_slow
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_5types11SlotStorageE17try_borrow_lockedB7_
-__RNvMs0_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_12BuiltinGuard3new
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13delete_subscr
-__RNvNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel7classes8pure_get
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtNtNtCs82bWklYMk3w_3std11collections4hash3set7HashSetyEE17try_borrow_lockedB7_
-__RNvMs_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB4_20ModuleFunctionsGuard3new
-__RNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB5_8JitState21resolve_native_callee
-__RNvMs1_NtCs2D882usVKTN_10weavepy_vm5tier2NtB5_8JitState19resolve_native_func
-__RNvXNtNtCs8Mbv00yxnRz_4core4iter8adaptersINtB2_12GenericShuntINtNtB2_3map3MapINtNtNtB6_5slice4iter4IterReENCNvMs_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB1N_20ModuleFunctionsGuard3new0EINtNtB6_6option6OptionNtNtB6_7convert10InfallibleEENtNtNtB4_6traits8iterator8Iterator4nextB1R_
-__RNvMs_NtCsblfCQaHOzq5_8indexmap5innerINtB4_4CoreNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBM_6ObjectE3popBO_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15pull_for_unpack
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtNtNtCs82bWklYMk3w_3std11collections4hash3map7HashMapyINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_8gc_trace13TrackedHandleEINtNtCs8Mbv00yxnRz_4core4hash18BuildHasherDefaultNtB2m_13GcIndexHasherEEE17try_borrow_lockedB7_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215method_ret_infos2_0B5_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync4WeakNtNtB7_6object11PyGeneratorEEE10borrow_mutB7_
-__RNvMs_NtCseWU5rFlE7Md_11weavepy_jit2irNtB4_9StrMethod9from_name
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtNtNtCshxvaOLs88l5_5alloc11collections5btree3map8BTreeMapyINtNtBZ_4sync3ArcNtNtB7_8gc_trace13TrackedHandleEEE17try_borrow_lockedB7_
-__RNvYNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel10make_dumpsINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtNtNtCs82bWklYMk3w_3std11collections4hash3map7HashMapyINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_8gc_trace13TrackedHandleEINtNtCs8Mbv00yxnRz_4core4hash18BuildHasherDefaultNtB2m_13GcIndexHasherEEE21try_borrow_mut_lockedB7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel10make_dumps
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215method_ret_infos1_0B5_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellANtNtB7_8gc_trace10Generationj3_E21try_borrow_mut_lockedB7_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215method_ret_infos6_0B5_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215method_ret_infos0_0B5_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtNtNtCs82bWklYMk3w_3std11collections4hash3set7HashSetyEE21try_borrow_mut_lockedB7_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm5tier215method_ret_infosa_0B5_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellAjj3_E21try_borrow_mut_lockedB7_
-__RNvXNtNtCs8Mbv00yxnRz_4core4iter8adaptersINtB2_12GenericShuntINtNtB2_3map3MapINtNtNtCsblfCQaHOzq5_8indexmap3map4iter4IterNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtB1X_6ObjectENCNvMs7_NtNtB1Z_6stdlib12pickle_accelNtB30_12PicklerGuard3new0EINtNtB6_6option6OptionNtNtB6_7convert10InfallibleEENtNtNtB4_6traits8iterator8Iterator4nextB1Z_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellAjj3_E17try_borrow_lockedB7_
-__RNvMs5_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_19ClassFunctionsGuard3new
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE21try_borrow_mut_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier220wpjit_iter_next_pair
-__RNvXs2_NtNtCs8Mbv00yxnRz_4core5slice3cmpNtNtCshxvaOLs88l5_5alloc6string6StringINtB5_14SlicePartialEqBC_E17equal_same_lengthCs2D882usVKTN_10weavepy_vm
-__RINvMNtCs8Mbv00yxnRz_4core6optionINtB3_6OptionNtNtCseWU5rFlE7Md_11weavepy_jit7analyze16MethodResolutionE7or_elseNCNvBK_10emit_instrs0_0EBM_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier220next_enumerated_byte
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtNtNtCshxvaOLs88l5_5alloc11collections5btree3map8BTreeMapyINtNtBZ_4sync3ArcNtNtB7_8gc_trace13TrackedHandleEEE21try_borrow_mut_lockedB7_
-__RNvYNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types22install_object_dunders14object_delattrINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215wpjit_bytes_get
-__RNvMs1_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit2ir14MethodSiteMetaEE6resizeB1l_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types22install_object_dunders14object_delattr
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecONtB7_11InterpreterEE17try_borrow_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier216wpjit_global_obj
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24generic_delattr_instance
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTmyEE8grow_oneCs2D882usVKTN_10weavepy_vm
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel19ClassFunctionsGuardE8grow_oneBS_
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit5lowerNtB2_7Lowerer22call_method_helper_sig
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214wpjit_call_dyn
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter30run_thread_local_death_cleanup
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_5types10TypeObjectEEE17try_borrow_lockedB7_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter24call_object_with_globals
-__RNvMNtCseWU5rFlE7Md_11weavepy_jit7analyzeNtB2_9PathArena3seg
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier27pin_any
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTINtNtCs8Mbv00yxnRz_4core6option6OptionmEmNtNtB7_6string6StringEE8grow_oneCseWU5rFlE7Md_11weavepy_jit
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared5pivot11median3_recNtNvMNtNtCs2jIxScAqd7x_9regalloc23ion5movesINtNtB1b_15data_structures3EnvpE34apply_allocations_and_insert_moves14InterBlockDestNCINvMB8_SB14_20sort_unstable_by_keyyNvMs_B16_B14_3keyE0ECs2EXTltNQfP1_17cranelift_codegen
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215attr_chain_step
-__RNvXs3_NtNtCs8Mbv00yxnRz_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCs2D882usVKTN_10weavepy_vm
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_16weakref_registry13RegistryInnerE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtB7_6object6ObjectB1o_EEE17try_borrow_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214wpjit_list_set
-__RNvNtCs2D882usVKTN_10weavepy_vm12type_surface15obj_len_builtin
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtBG_6string6StringNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEEB1y_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface19install_type_checks18type_subclasscheck
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs2D882usVKTN_10weavepy_vm11InterpreterEBD_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm6import11ModuleCacheEBF_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins18mappingproxy_itemsINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins18mappingproxy_items
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueTmNtNtCs2D882usVKTN_10weavepy_vm5tier29SynthSeedEEBH_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBG_5boxed3BoxNtCs2D882usVKTN_10weavepy_vm9InlineActEEEB1s_
-__RNvCs2D882usVKTN_10weavepy_vm16fallback_globals
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtCs2D882usVKTN_10weavepy_vm9InlineActEEB1c_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_mod5build
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtCs2D882usVKTN_10weavepy_vm12shared_value7ThinArcNtNtB1d_13tuple_storage12TupleStorageEEEB1d_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm13builtin_types14make_owned_news_0B5_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object18native_scalar_repr
-__RNvNvNtCs2D882usVKTN_10weavepy_vm13builtin_types14make_owned_new11reject_bool
-__RNvXso_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecIBw_NtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEENtNtNtCs8Mbv00yxnRz_4core3ops4drop4Drop4dropBN_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_4cell10UnsafeCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB15_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm6object10FrameShellEEEEB1T_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier216wpjit_build_list
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier217wpjit_list_repeat
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier213wpjit_call_py
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5tier211MethodEntryEBF_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtCs2D882usVKTN_10weavepy_vm8LeafRecvReINtNtBG_4sync3ArcNtNtB1b_6object9BuiltinFnEEEEB1b_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215try_native_call
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier217enter_scalar_leaf
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier214wpjit_list_get
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons23clear_thread_python_tls
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB1c_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtB7_6object6ObjectB1o_EEE21try_borrow_mut_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm13builtin_types26clear_thread_type_registry
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm3gil8GilGuardEBF_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecAmj40_E21reserve_one_uncheckedCs2EXTltNQfP1_17cranelift_codegen
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10signal_mod25unregister_thread_pthread
-__RNvCs2D882usVKTN_10weavepy_vm13slice_indices
-__RNvNtCs2D882usVKTN_10weavepy_vm13vm_singletons22clear_worker_thread_id
-__RINvMs_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectE5drainINtNtNtCs8Mbv00yxnRz_4core3ops5range14RangeInclusivejEEBJ_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6result6ResultNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNtNtB11_5error12RuntimeErrorEEB11_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtCs8Mbv00yxnRz_4core6option6OptionIBx_NtNtCs2D882usVKTN_10weavepy_vm15thread_registry11ThreadEntryEEEE9drop_slowB2S_
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtB1a_6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEEEEB2F_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_ordINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst10blockorder12LoweredBlockj80_E21reserve_one_uncheckedBO_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_ord
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellNtNtCs2D882usVKTN_10weavepy_vm5tier27JitBufsEEB1J_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecANtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer12MachCallSitej10_E21reserve_one_uncheckedBO_
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecAINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer10MachSrcLocNtBL_7StencilEj40_E21reserve_one_uncheckedBP_
-__RNvNtNtNtNtCs2EXTltNQfP1_17cranelift_codegen3isa7aarch644inst4emit23enc_test_bit_and_branch
-__RNvMsc_CsfNVduRRWYaN_8smallvecINtB5_8SmallVecAINtNtNtCs2EXTltNQfP1_17cranelift_codegen8machinst6buffer10MachSrcLocNtBL_5FinalEj40_E8try_growBP_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtB7_3vec3VecIBx_NtNtBL_6object10FrameShellEEEE9drop_slowBL_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtB7_3vec3VecNtNtBL_5error11PyExceptionEEE9drop_slowBL_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm3gil8GilGuardEEB1c_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNCINvMsZ_NtCs2D882usVKTN_10weavepy_vm6objectNtBK_9BuiltinFn11with_kwargsNCNvNtNtBM_6stdlib11thread_real16make_lock_object0E0EBM_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real16make_lock_objects0_0EBJ_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtCs2D882usVKTN_10weavepy_vm4sync8RealLockE9drop_slowBK_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object13int_from_i128
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real25make_thread_handle_objects_0B7_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17do_list_sort_call
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionTyINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_5types12InstancePlanEEEE17try_borrow_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionTyINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_5types12InstancePlanEEEE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object6ObjectE17try_borrow_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object6ObjectE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object12LazyIterKindE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object12LazyIterKindE17try_borrow_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync4WeakNtNtB7_5types10PyInstanceEEE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCsblfCQaHOzq5_8indexmap3set8IndexSetNtNtB7_6object7DictKeyNtNtB7_8fasthash13FxBuildHasherEE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBV_4sync3ArcNtNtB7_5types10TypeObjectEEE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBV_4sync4WeakNtNtB7_5types10TypeObjectEEE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_5types10TypeObjectEEE21try_borrow_mut_lockedB7_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCseWU5rFlE7Md_11weavepy_jit2ir5TFuncECs2D882usVKTN_10weavepy_vm
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel10make_dumps0B7_
-__RNvMs5_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_19ClassFunctionsGuard5holds
-__RNvMs2_NtCshxvaOLs88l5_5alloc7raw_vecNtB5_11RawVecInner11try_reserveCs2D882usVKTN_10weavepy_vm
-__RNvMs_NtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encodeNtB4_7Encoder4save
-__RNvMNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encodeNtB2_6Framer6commit
-__RINvMsj_NtCshxvaOLs88l5_5alloc3vecINtB6_3VecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectBH_EE16extend_desugaredINtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtNtCsblfCQaHOzq5_8indexmap3map4iter4IterNtBJ_7DictKeyBH_ENCNvMs_NtNtNtBL_6stdlib12pickle_accel6encodeNtB3G_7Encoder4save0EEBL_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTxuEE14reserve_rehashNCINvNtB8_3map11make_hasherxuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0ECs2D882usVKTN_10weavepy_vm
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapxuNtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertCs2D882usVKTN_10weavepy_vm
-__RNvMNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encodeNtB2_6Framer5write
-__RNvMs_NtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encodeNtB4_7Encoder7memoize
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTxTmNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEE14reserve_rehashNCINvNtB8_3map11make_hasherxBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EBX_
-__RNvMs_NtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encodeNtB4_7Encoder10save_items
-__RNvMNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encodeNtB2_6Framer4data
-__RNvMs_NtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encodeNtB4_7Encoder7integer
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectB19_EEEB1d_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encode15InstanceContextEEB15_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTINtNtBG_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm5types10TypeObjectEIBC_NtNtB1u_6object6ObjectEEEEB1u_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel10make_loads0B7_
-__RNvMs4_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_14UnpicklerGuard5holds
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel5parseINtB2_5ProbeKb0_EEB6_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel4pushNtB2_4KindEB6_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel4pushjEB6_
-__RNvXsc_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelINtB5_5ProbeKb0_ENtB5_4Sink6extendB9_
-__RNvXsc_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelINtB5_5ProbeKb0_ENtB5_4Sink8setitemsB9_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel4pushmEB6_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins14int_bit_lengthINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel4takeNtB2_4KindEB6_
-__RNvXsc_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelINtB5_5ProbeKb0_ENtB5_4Sink5tupleB9_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins14int_bit_length
-__RNvMsn_NtCsfGIcTr55AEy_10num_bigint6bigintNtB5_6BigInt3pow
-__RNvXsa_NtNtCsfGIcTr55AEy_10num_bigint7biguint5powerNtB7_7BigUintINtNtCshMjN7IvaumL_10num_traits3pow3PowmE3pow
-__RNvXs8_NtNtCsfGIcTr55AEy_10num_bigint7biguint14multiplicationRNtB7_7BigUintNtNtNtCs8Mbv00yxnRz_4core3ops5arith3Mul3mul
-__RNvMsb_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelINtB5_5ProbeKb0_E19acyclic_and_boundedB9_
-__RNvNtNtCsfGIcTr55AEy_10num_bigint7biguint14multiplication10scalar_mul
-__RNvMs1_NtCshxvaOLs88l5_5alloc3vecINtB5_3VechE6resizeCs2D882usVKTN_10weavepy_vm
-__RNvXsa_NtNtCsfGIcTr55AEy_10num_bigint7biguint14multiplicationNtB7_7BigUintINtNtNtCs8Mbv00yxnRz_4core3ops5arith9MulAssignRBX_E10mul_assign
-__RNCNvMs2_Cs2D882usVKTN_10weavepy_vmNtB7_11Interpreter4steps3_0B7_
-__RNvMsb_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelINtB5_5ProbeKb0_E4planB9_
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort11insert_tailNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectNCINvMNtCshxvaOLs88l5_5alloc5sliceSB18_7sort_byNCNvMs2_B1c_NtB1c_11Interpreter13sort_with_keys3_0E0EB1c_
-__RNvYNvNvMNtCs2D882usVKTN_10weavepy_vm13builtin_typesNtB7_12BuiltinTypes5build9proxy_newINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB9_6object6ObjectEE4callB9_
-__RNvNvMNtCs2D882usVKTN_10weavepy_vm13builtin_typesNtB4_12BuiltinTypes5build9proxy_new
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_chrINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_chr
-__RNvNtCs2D882usVKTN_10weavepy_vm6object16range_len_bigint
-__RNvMsn_NtCs2D882usVKTN_10weavepy_vm6objectNtB5_5Range6bounds
-__RNvXs0_NtNtCsfGIcTr55AEy_10num_bigint6bigint11subtractionNtB7_6BigIntINtNtNtCs8Mbv00yxnRz_4core3ops5arith3SubRBT_E3sub
-__RNvMs1_NtCseaAvxSPddma_11parking_lot9raw_mutexNtB5_8RawMutex11unlock_slow
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real20join_wait_collecting
-__RNvCs2D882usVKTN_10weavepy_vm11range_slice
-__RNCNvCs2D882usVKTN_10weavepy_vm11range_slice0B3_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc4sync3ArcIBD_NtNtB7_6object8DictDataEEE17try_borrow_lockedB7_
-__RNvXs2_NtNtCsfGIcTr55AEy_10num_bigint6bigint7convertNtB7_6BigIntINtNtCs8Mbv00yxnRz_4core7convert4FromxE4from
-__RNvXNtNtCsfGIcTr55AEy_10num_bigint6bigint11subtractionRNtB4_6BigIntNtNtNtCs8Mbv00yxnRz_4core3ops5arith3Sub3sub
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyNtNtCs6I8f3QNB0wx_16parking_lot_core11parking_lot10ThreadDataECseaAvxSPddma_11parking_lot
-__RNCNvCs2D882usVKTN_10weavepy_vm11range_slices_0B3_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel4NodeEEB1e_
-__RNvXsf_NtNtCsfGIcTr55AEy_10num_bigint6bigint14multiplicationRNtB7_6BigIntNtNtNtCs8Mbv00yxnRz_4core3ops5arith3Mul3mul
-__RNvXs_NtNtCsfGIcTr55AEy_10num_bigint6bigint8additionRNtB6_6BigIntINtNtNtCs8Mbv00yxnRz_4core3ops5arith3AddBP_E3add
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier217wpjit_call_method
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter15do_min_max_call
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter10op_compare
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter21deep_order_collection
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel5parseNtB2_7ObjectsEB6_
-__RNvCs2D882usVKTN_10weavepy_vm12is_nan_value
-__RNvXse_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_7ObjectsNtB5_4Sink5empty
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel4pushNtNtB6_6object6ObjectEB6_
-__RNvXse_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_7ObjectsNtB5_4Sink6scalar
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier218wpjit_dyn_attr_get
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_mod11sre_compile
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel5batchNtNtB6_6object6ObjectEB6_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_mod14codeseq_to_vec0B7_
-__RNvXse_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_7ObjectsNtB5_4Sink6extend
-__RNvXse_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_7ObjectsNtB5_4Sink8setitems
-__RNvMs_NtCsblfCQaHOzq5_8indexmap5innerINtB4_4CoreNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBM_6ObjectE11try_reserveBO_
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB2q_4sync3ArcNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_mod12CompiledCodeEEEE10initializeNCINvB2_11get_or_initNCNvB3c_8registry0E0zEB3g_
-__RNCINvMs0_NtNtCs82bWklYMk3w_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs1jOiKDdKJOr_8lock_api5mutex5MutexNtNtCseaAvxSPddma_11parking_lot9raw_mutex8RawMutexINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtB3d_4sync3ArcNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_mod12CompiledCodeEEEE10initializeNCINvB1a_11get_or_initNCNvB3Z_8registry0E0zE0E0B43_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel4takeNtNtB6_6object6ObjectEB6_
-__RNvXse_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_7ObjectsNtB5_4Sink5tuple
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecINtNtB7_4sync3ArcNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_mod12CompiledCodeEE8grow_oneB19_
-__RINvMs3_NtCsblfCQaHOzq5_8indexmap3mapINtB6_8IndexMapNtNtCs2D882usVKTN_10weavepy_vm6object7DictKeyNtBQ_6ObjectNtNtBS_8fasthash13FxBuildHasherE7get_mutNtBQ_12StrKeyHashedEBS_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_mod8sre_exec
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel7ObjectsEBH_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_mod14subject_to_vec
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionTjNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtB1M_3vec3VecmEEEEEB13_
-__RNvMs_NtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encodeNtB4_7Encoder13save_instance
-__RNvMNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_modNtB2_7Matcher3new
-__RNCNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel10make_dumps00B9_
-__RNvMs1_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_11ImportGuard7modules
-__RNvMNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_modNtB2_7Matcher8do_match
-__RNvMNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_modNtB2_7Matcher14do_match_inner
-__RNvMs0_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_12BuiltinGuard5holds
-__RNvMs_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB4_20ModuleFunctionsGuard5holds
-__RNvNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel7classes12plain_layout
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_mod7MatcherEBH_
-__RNvNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel7classes13is_heap_class
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtNtCs2D882usVKTN_10weavepy_vm5tier27CallCtxEEB1e_
-__RNvNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel7classes16benign_metaclass
-__RNvNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel7classes14resolve_global
-__RNvNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel7classes10slot_names
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encode9ClassPlanEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1x_
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapjINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encode9ClassPlanENtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertB1u_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier221finish_deopted_callee
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel5build
-__RNvNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel7classes14is_member_slot
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel1b
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel6encode9ClassPlanE9drop_slowBO_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier213rebuild_stack
-__RNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface19install_type_checks18type_instancecheck
-__RNvXsc_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelINtB5_5ProbeKb0_ENtB5_4Sink6globalB9_
-__RNCNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel10make_loads00B9_
-__RINvMs2_NtNtCs82bWklYMk3w_3std6thread5localINtB6_8LocalKeyINtNtCs8Mbv00yxnRz_4core4cell7RefCellNtNtCs2D882usVKTN_10weavepy_vm5tier28JitStateEE4withNCNvB1x_16try_enter_resume0INtNtBZ_6option6OptionNtB1x_13CompiledEntryEEB1z_
-__RNvMs9_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_10PlainClass7resolve
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared9smallsort11insert_tailTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectB19_ENCINvMNtCshxvaOLs88l5_5alloc5sliceSB18_7sort_byNCNvMs2_B1d_NtB1d_11Interpreter13sort_with_keys0_0E0EB1d_
-__RNvXsc_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelINtB5_5ProbeKb0_ENtB5_4Sink6newobjB9_
-__RNvXsc_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelINtB5_5ProbeKb0_ENtB5_4Sink5buildB9_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accel4pushTNtB2_10StateNodesINtNtCs8Mbv00yxnRz_4core6option6OptionmEEEB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier221defer_activation_pins
-__RNvXse_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_7ObjectsNtB5_4Sink6newobj
-__RNvCs2D882usVKTN_10weavepy_vm15exception_value
-__RNvNtCs2D882usVKTN_10weavepy_vm6object10repr_enter
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtCs2D882usVKTN_10weavepy_vm5FrameEBD_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecTmNtNtCs2D882usVKTN_10weavepy_vm5error11PyExceptionEEEB1e_
-__RNvXsq_NtCshxvaOLs88l5_5alloc6stringNtB5_6StringNtNtCs8Mbv00yxnRz_4core3fmt7Display3fmt
-__RNvXse_NtNtCs2D882usVKTN_10weavepy_vm6stdlib12pickle_accelNtB5_7ObjectsNtB5_4Sink5build
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCs2D882usVKTN_10weavepy_vm5error11PyExceptionEEB1c_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object10repr_leave
-__RNvXs2_NtNtCsfGIcTr55AEy_10num_bigint6bigint7convertNtB7_6BigIntINtNtCs8Mbv00yxnRz_4core7convert4FromxE4from
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCshxvaOLs88l5_5alloc5boxed3BoxNtNtCs2D882usVKTN_10weavepy_vm5tier216NativeActivationEEEB1A_
-__RNvXs0_NtNtCsfGIcTr55AEy_10num_bigint6bigint8additionNtB7_6BigIntINtNtNtCs8Mbv00yxnRz_4core3ops5arith3AddRBP_E3add
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5tier212NativeCalleeEBF_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib3sys13intern_shared
-__RNvXsc_NtNtCsfGIcTr55AEy_10num_bigint6bigint11subtractionNtB7_6BigIntINtNtNtCs8Mbv00yxnRz_4core3ops5arith3SublE3sub
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier218finish_interp_call
-__RNvXs2_NtNtCsfGIcTr55AEy_10num_bigint7biguint11subtractionNtB7_7BigUintINtNtNtCs8Mbv00yxnRz_4core3ops5arith9SubAssignmE10sub_assign
-__RNvNtNtCsfGIcTr55AEy_10num_bigint7biguint11subtraction4sub2
-__RNvXsF_NtNtCsfGIcTr55AEy_10num_bigint6bigint8divisionNtB7_6BigIntNtNtNtCs8Mbv00yxnRz_4core3ops5arith3Div3div
-__RNvMs6_NtCshxvaOLs88l5_5alloc2rcINtB5_2RcNtNtCs2D882usVKTN_10weavepy_vm5tier211DirectEntryE9drop_slowBH_
-__RNvNtNtCsfGIcTr55AEy_10num_bigint7biguint8division11div_rem_ref
-__RNvXs_NtNtCshxvaOLs88l5_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs8Mbv00yxnRz_4core4iter8adapters3map3MapINtNtB2m_3rev3RevINtNtNtB2q_5slice4iter4IterNtNtB17_7analyze2SEEENCNvB3P_13step_abstracts4_0EE9from_iterB17_
-__RNvXs0_NtNtCsfGIcTr55AEy_10num_bigint6bigint11subtractionNtB7_6BigIntINtNtNtCs8Mbv00yxnRz_4core3ops5arith3SubRBT_E3sub
-__RNvMs1_NtCs3fRkGY2FHAU_9hashbrown3mapINtB5_7HashMapmTmINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEENtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE6insertB1r_
-__RNvXs0_NtNtCsfGIcTr55AEy_10num_bigint6bigint8additionNtB7_6BigIntINtNtNtCs8Mbv00yxnRz_4core3ops5arith3AddRBP_E3add
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTmTmINtNtCshxvaOLs88l5_5alloc3vec3VecNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1u_
-__RNCNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface21install_value_richcmp15richcmp_builtin0B7_
-__RNvNvNtCs2D882usVKTN_10weavepy_vm12type_surface21install_value_richcmp7fam_int
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins13dict_fromkeysINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins13dict_fromkeys
-__RNvXNtNtCsfGIcTr55AEy_10num_bigint7biguint11subtractionNtB4_7BigUintINtNtNtCs8Mbv00yxnRz_4core3ops5arith3SubRBR_E3sub
-__RNvXs_NtNtCsfGIcTr55AEy_10num_bigint7biguint11subtractionNtB6_7BigUintINtNtNtCs8Mbv00yxnRz_4core3ops5arith9SubAssignRBT_E10sub_assign
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11list_removeINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter17make_seq_iterator
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11list_remove
-__RNvNtCs2D882usVKTN_10weavepy_vm6object9member_eq
-__RINvNtNtNtCs8Mbv00yxnRz_4core5slice4sort8unstable7ipnsortTjjNtNtCs8MhVFxlpMWQ_16cranelift_module6module6FuncIdENCINvMB6_SBT_20sort_unstable_by_keyjNCNvMs0_NtCslsAwG9FmMO9_13cranelift_jit7backendNtB2t_9JITModule20finalize_definitionss0_0E0EB2v_
-__RNvNtCseWU5rFlE7Md_11weavepy_jit7analyze17unary_result_type
-__RNCNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15analyze_for_rets2_0B5_
-__RNCNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15analyze_for_rets1_0B5_
-__RNCNvNtCseWU5rFlE7Md_11weavepy_jit7analyze15analyze_for_rets8_0B5_
-__RNvCs2D882usVKTN_10weavepy_vm8unary_op
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier212wpjit_str_eq
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier213wpjit_str_get
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier215wpjit_const_str
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier218wpjit_dyn_attr_set
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtB7_6object6ObjectEE21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync4WeakNtNtB7_5types10PyInstanceEEE17try_borrow_lockedB7_
-__RNvMs4_NtCshxvaOLs88l5_5alloc7raw_vecINtB5_6RawVecTNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectBN_EE8grow_oneBR_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellmE21try_borrow_mut_lockedB7_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11set_discardINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11set_discard
-__RINvMs10_NtNtNtCshxvaOLs88l5_5alloc11collections5btree4nodeINtB7_16BalancingContextyINtNtBd_4sync3ArcNtNtCs2D882usVKTN_10weavepy_vm8gc_trace13TrackedHandleEE25merge_tracking_child_edgeNtNtBd_5alloc6GlobalEB1F_
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier221native_exit_writeback
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object11FileBackendE17try_borrow_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCshxvaOLs88l5_5alloc6string6StringEE17try_borrow_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellDNtNtCs82bWklYMk3w_3std2io5WriteNtNtCs8Mbv00yxnRz_4core6marker4SyncNtB1o_4SendEL_E21try_borrow_mut_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_6object7PyFrameEEE17try_borrow_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCs8Mbv00yxnRz_4core6option6OptionINtNtCshxvaOLs88l5_5alloc4sync3ArcNtNtB7_6object7PyFrameEEE21try_borrow_mut_lockedB7_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real25make_thread_handle_object0B7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE17try_borrow_lockedB7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real19is_main_interpreter
-__RINvNtNtNtNtCs8Mbv00yxnRz_4core5slice4sort6shared5pivot11median3_recjNvYjNtNtBa_3cmp10PartialOrd2ltECseWU5rFlE7Md_11weavepy_jit
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real25make_thread_handle_objects0_0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib11thread_real15thread_shutdown
-__RNvMs0_NtCs2D882usVKTN_10weavepy_vm15thread_registryNtB5_14ThreadRegistry15join_non_daemon
-__RNvNtCs2D882usVKTN_10weavepy_vm5tier216probe_ctor_field
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNtNtCs2D882usVKTN_10weavepy_vm5tier214ClassCtorEntryEBF_
-__RNvMs1_NtNtCs82bWklYMk3w_3std6thread9lifecycleINtB5_9JoinInneruE4joinCs2D882usVKTN_10weavepy_vm
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtNtCs82bWklYMk3w_3std6thread9lifecycle6PacketuEE9drop_slowCs2D882usVKTN_10weavepy_vm
-__RNCNvNtCs2D882usVKTN_10weavepy_vm8builtins21bytes_from_source_obj0B5_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins18bytearray_byte_arg
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins10bytes_findINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtCs8Mbv00yxnRz_4core6option6OptionTNtNtB7_6object6ObjectIBR_B20_EIBR_TNtNtBV_6string6StringB20_EEEEEE17try_borrow_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins10bytes_find
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins20bytes_needle_guarded
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtCs8Mbv00yxnRz_4core6option6OptionTNtNtB7_6object6ObjectIBR_B20_EIBR_TNtNtBV_6string6StringB20_EEEEEE21try_borrow_mut_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object24bytearray_export_acquire
-__RINvMNtNtCs82bWklYMk3w_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexINtNtNtNtB7_11collections4hash3map7HashMapjTINtNtCshxvaOLs88l5_5alloc4sync4WeakINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtB29_3vec3VechEEEjEEEE10initializeNCINvB2_11get_or_initNCNvNtB2I_6object20bytearray_export_map0E0zEB2I_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecyEE17try_borrow_lockedB7_
-__RINvMs6_NtCs3fRkGY2FHAU_9hashbrown3rawINtB6_8RawTableTjTINtNtCshxvaOLs88l5_5alloc4sync4WeakINtNtCs2D882usVKTN_10weavepy_vm4sync7GilCellINtNtBX_3vec3VechEEEjEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs82bWklYMk3w_3std4hash6random11RandomStateE0EB1w_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object20ByteArrayExportGuardEEB11_
-__RNvNtCs2D882usVKTN_10weavepy_vm6object24bytearray_export_release
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native4lazy7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapPNtCs6MP6LIxl4z2_16weavepy_compiler10CodeObjectTINtNtB19_6option6OptionNtNtCseWU5rFlE7Md_11weavepy_jit5value7JitTypeEbINtNtCshxvaOLs88l5_5alloc4sync3ArcB2m_EEEEECs2D882usVKTN_10weavepy_vm
-__RNvNtCsc1KvXAN9vgu_6memchr6memmem4find
-__RINvMNtNtCsc1KvXAN9vgu_6memchr6memmem8searcherNtB3_8Searcher3newNtNtNtNtB7_4arch3all10packedpair20DefaultFrequencyRankECs2D882usVKTN_10weavepy_vm
-__RNvNtNtCsc1KvXAN9vgu_6memchr6memmem8searcher22searcher_kind_one_byte
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellANtNtB7_8gc_trace10Generationj3_E17try_borrow_lockedB7_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19dispatch_inplace_op
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecINtNtBV_4sync3ArcNtNtB7_8gc_trace13TrackedHandleEEE17try_borrow_lockedB7_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellbE17try_borrow_lockedB7_
-__RNCNvNtCs2D882usVKTN_10weavepy_vm8builtins14numeric_dunders2_0B5_
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter13do_float_call
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VecTNtNtBV_6string6StringNtNtB7_6object6ObjectEEE17try_borrow_lockedB7_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins7b_float
-__RNCINvNtCs2D882usVKTN_10weavepy_vm8builtins9method_kwNvB4_18bytes_translate_kwEs_0B6_
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellINtNtCshxvaOLs88l5_5alloc3vec3VechEE21try_borrow_mut_lockedB7_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins18bytes_translate_kwINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCshxvaOLs88l5_5alloc6string6StringB1I_EEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins13bytes_argview
-__RNvMsf_NtCs2D882usVKTN_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object11FileBackendE21try_borrow_mut_lockedB7_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNvNtCs2D882usVKTN_10weavepy_vm8builtins18bytes_translate_kwE9drop_slowBK_
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins11list_insertINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins11list_insert
-__RNvYNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_strINtNtNtCs8Mbv00yxnRz_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
-__RNvNtCs2D882usVKTN_10weavepy_vm8builtins5b_str
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter19format_template_str
-__RNvCs2D882usVKTN_10weavepy_vm17scan_format_field
-__RNvCs2D882usVKTN_10weavepy_vm18split_format_field
-__RNvCs2D882usVKTN_10weavepy_vm19split_name_trailers
-__RNvCs2D882usVKTN_10weavepy_vm14format_int_hex
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNvNtCs2D882usVKTN_10weavepy_vm8builtins13str_format_kwE9drop_slowBK_
-__RNvYFG_RL0_SNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEINtNtCs8Mbv00yxnRz_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel17json_make_scanner
-__RNvMs2_Cs2D882usVKTN_10weavepy_vmNtB5_11Interpreter8op_truth
-__RNvCs2D882usVKTN_10weavepy_vm18long_overflows_f64
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel5builds_0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel17json_make_encoder
-__RNvMs1_NtCshxvaOLs88l5_5alloc3vecINtB5_3VecINtNtCs8Mbv00yxnRz_4core6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEE6resizeB1l_
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueSINtNtB4_6option6OptionNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectEEB12_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel11with_interpNCNvB2_17json_make_encoders2_0TbbbEEB6_
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel17json_make_encoders5_0B7_
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel12encode_value
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel17encode_value_impl
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel9json_kind
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accelNtB5_10JsonWriter8push_str
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel20write_encoded_string
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accelNtB5_10JsonWriter11quoted_utf8
-__RNvMs3_NtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accelNtB5_10JsonWriter11push_object
-__RNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel11encode_list0B7_
-__RNvXs_NtCsl6mUoftqY9j_4zmij7privatedNtB4_6Sealed20write_to_zmij_buffer
-__RINvNtCs8Mbv00yxnRz_4core3ptr9drop_glueNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel17json_make_encoders5_0EBJ_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcNtNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel10EncoderCfgE9drop_slowBM_
-__RNvMsn_NtCshxvaOLs88l5_5alloc4syncINtB5_3ArcINtNtB7_3vec3VecmEE9drop_slowCs2D882usVKTN_10weavepy_vm
-__RNvMNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_modNtB2_7Matcher5count
-__RNvMNtNtCs2D882usVKTN_10weavepy_vm6stdlib7sre_modNtB2_7Matcher7charset
-__RNCNCNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel17json_make_scanners_00B9_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel9scan_oncehEB6_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel12parse_objecthEB6_
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel21scan_string_with_memohEB6_
-__RNvXNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accelhNtB2_8JsonChar3key
-__RINvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel11parse_arrayhEB6_
-__RNvXNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accelhNtB2_8JsonChar6string
-__RNvNtNtCs2D882usVKTN_10weavepy_vm6stdlib10json_accel13finish_object
-__RINvNtNtNtNtCs82bWklYMk3w_3std3sys12thread_local6native5eager7destroyINtNtCs8Mbv00yxnRz_4core4cell7RefCellINtNtB1a_6option6OptionTjNtNtCs2D882usVKTN_10weavepy_vm6object6ObjectINtNtCshxvaOLs88l5_5alloc4sync3ArcINtNtB2S_3vec3VecmEEEEEEB29_
+__RNvCsdtSMxSA38aw_7___rustc12___rust_alloc
+__RNvCsljrJvzJKKaT_11weavepy_cli8cli_main
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10signal_mod12StoredSigsetE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEBZ_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10signal_mod12StoredSigsetE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B1M_
+__RNvCsdtSMxSA38aw_7___rustc14___rust_realloc
+__RNvCsdtSMxSA38aw_7___rustc14___rust_dealloc
+__RNSNvYNCINvNtNtCs6KxzipSu1yP_3std6thread9lifecycle15spawn_uncheckedNCNvCsljrJvzJKKaT_11weavepy_cli18run_on_large_stack0lEs_0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceuE9call_once6vtableB17_
+__RINvNtNtCs6KxzipSu1yP_3std3sys9backtrace28___rust_begin_short_backtraceNCNvCsljrJvzJKKaT_11weavepy_cli18run_on_large_stack0lEB1b_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10signal_mod36unblock_async_signals_current_thread
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10signal_mod28install_startup_dispositions
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtNtBa_6poison5mutex5MutexINtNtNtNtBc_11collections4hash3map7HashMaplNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEE10initializeNCINvB1a_11get_or_initNCNvNtNtB2U_6stdlib10signal_mod8handlers0E0zE0E0B2U_
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell4CellTyyEEE4withNCNvMNtNtBa_4hash6randomNtB1I_11RandomState3new0B21_ECs4ZIJX5XqtQk_11weavepy_jit
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvNtNtB1K_6stdlib10signal_mod23default_int_handler_obj0E0zE0E0B1K_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10signal_mod1b
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMaplNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBS_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRlECs5bNXvwCVaF0_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTlNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherlBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCseIGkHRfguhS_9hashbrown10scopeguard10ScopeGuardNtNtBG_3raw13RawTableInnerNCINvMsa_B1u_B1s_14prepare_resizeNtNtCsdq5mISKDh0s_5alloc5alloc6GlobalE0EECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10signal_mod11set_handler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEBF_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10signal_mod18set_os_disposition
+__RNvCsljrJvzJKKaT_11weavepy_cli13main_dispatch
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtNtCsjmumdDSMH2I_18tracing_subscriber6filter3env7builder7BuilderECsljrJvzJKKaT_11weavepy_cli
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE10initializeNCINvB2_11get_or_initNCNvNtNtBX_6stdlib10signal_mod23default_int_handler_obj0E0zEBX_
+__RNvMNtNtNtCsjmumdDSMH2I_18tracing_subscriber6filter3env9directiveNtB2_9Directive5parse
+__RNvXse_NtCskEISuMjXst2_12tracing_core8metadataNtB5_11LevelFilterNtNtNtCs2NHf1dqD6Ob_4core3str6traits7FromStr8from_str
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtNtCsjmumdDSMH2I_18tracing_subscriber6filter3env9directive9DirectiveECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCsjmumdDSMH2I_18tracing_subscriber6filter3env12FromEnvErrorECsljrJvzJKKaT_11weavepy_cli
+__RNvXsb_NtNtNtCs6KxzipSu1yP_3std4sync6poison6rwlockINtB5_6RwLockINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskEISuMjXst2_12tracing_core10dispatcher9RegistrarEENtNtCs2NHf1dqD6Ob_4core7default7Default7defaultB1A_
+__RNvCsljrJvzJKKaT_11weavepy_cli12bridged_args
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtNtCsjmumdDSMH2I_18tracing_subscriber6filter3env9directive9DirectiveE8grow_oneBU_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsjmumdDSMH2I_18tracing_subscriber
+__RNvMNtNtNtCsjmumdDSMH2I_18tracing_subscriber6filter3env9directiveNtB2_9Directive9to_static
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtNtCsjmumdDSMH2I_18tracing_subscriber6filter3env5field5MatchEECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtNtCsjmumdDSMH2I_18tracing_subscriber6filter3env9directive9DirectiveEECsljrJvzJKKaT_11weavepy_cli
+__RINvMs2_NtCs2kTwstJmECD_9once_cell3impINtB6_8OnceCellINtNtNtNtCs6KxzipSu1yP_3std4sync6poison6rwlock6RwLockINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskEISuMjXst2_12tracing_core10dispatcher9RegistrarEEE10initializeNCINvMs4_NtB8_4syncINtB3t_8OnceCellBP_E11get_or_initNCNvMs9_B3t_INtB3t_4LazyBP_E5force0E0NtNvMs4_B3t_IB3E_pE11get_or_init4VoidEB2h_
+__RNvNtCs2kTwstJmECD_9once_cell3imp18initialize_or_wait
+__RNCINvMs2_NtCs2kTwstJmECD_9once_cell3impINtB8_8OnceCellINtNtNtNtCs6KxzipSu1yP_3std4sync6poison6rwlock6RwLockINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskEISuMjXst2_12tracing_core10dispatcher9RegistrarEEE10initializeNCINvMs4_NtBa_4syncINtB3v_8OnceCellBR_E11get_or_initNCNvMs9_B3v_INtB3v_4LazyBR_E5force0E0NtNvMs4_B3v_IB3G_pE11get_or_init4VoidE0B2j_
+__RNvXs3_NtCs2kTwstJmECD_9once_cell3impNtB5_5GuardNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCskEISuMjXst2_12tracing_core10dispatcher9RegistrarE8grow_oneBQ_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner14grow_amortizedCskEISuMjXst2_12tracing_core
+__RNvYINtNtCsjmumdDSMH2I_18tracing_subscriber3fmt10SubscriberNtNtB5_6format13DefaultFieldsNtBX_6FormatNtNtNtB7_6filter3env9EnvFilterENtNtCskEISuMjXst2_12tracing_core10subscriber10Subscriber20on_register_dispatchCsljrJvzJKKaT_11weavepy_cli
+__RNvXs0_NtCsjmumdDSMH2I_18tracing_subscriber3fmtINtB5_10SubscriberNtNtB5_6format13DefaultFieldsNtB13_6FormatNtNtNtB7_6filter3env9EnvFilterENtNtCskEISuMjXst2_12tracing_core10subscriber10Subscriber14max_level_hintCsljrJvzJKKaT_11weavepy_cli
+__RNvXNtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtB6_6string6StringEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters3map3MapNtNtCs6KxzipSu1yP_3std3env6ArgsOsNCNvCs5bNXvwCVaF0_10weavepy_vm15os_args_bridged0EE9from_iterB3g_
+__RNvNvCs5bNXvwCVaF0_10weavepy_vm15os_args_bridged21register_if_ambiguous
+__RNvCsljrJvzJKKaT_11weavepy_cli9real_main
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec14spec_from_iterINtB6_3VecNtNtB8_6string6StringEINtB4_12SpecFromIterBW_INtNtB6_9into_iter8IntoIterBW_EE9from_iterCsljrJvzJKKaT_11weavepy_cli
+__RNvXs3_CsljrJvzJKKaT_11weavepy_cliNtB5_3CliNtNtCskkb32ezpab6_12clap_builder6derive4Args12augment_args
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCskkb32ezpab6_12clap_builder4util2id2IdE8grow_oneBS_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCskkb32ezpab6_12clap_builder
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCskkb32ezpab6_12clap_builder7builder9arg_group8ArgGroupE8grow_oneBS_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtCskkb32ezpab6_12clap_builder7builder12value_parser11ValueParserEEB13_
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECskkb32ezpab6_12clap_builder
+__RNvMs4_NtCskkb32ezpab6_12clap_builder7mkeymapNtB5_7MKeyMap4push
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCskkb32ezpab6_12clap_builder7builder3arg3ArgE8grow_oneBS_
+__RINvMs_NtNtCskkb32ezpab6_12clap_builder7builder3argNtB5_3Arg12value_parserINtNtB7_12value_parser20RangedI64ValueParserhEECsljrJvzJKKaT_11weavepy_cli
+__RINvMs0_NtNtCskkb32ezpab6_12clap_builder7builder7commandNtB6_7Command5aboutReEBa_
+__RINvMNtNtCskkb32ezpab6_12clap_builder7builder7commandNtB3_7Command16get_matches_fromINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB1p_6string6StringEB1R_ECsljrJvzJKKaT_11weavepy_cli
+__RNvMs3_NtNtCskkb32ezpab6_12clap_builder7builder7commandNtB5_7Command9__do_parse
+__RNvMs3_NtNtCskkb32ezpab6_12clap_builder7builder7commandNtB5_7Command11__build_self
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCskkb32ezpab6_12clap_builder7builder7command7CommandEEB1e_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCskkb32ezpab6_12clap_builder7mkeymap3KeyE8grow_oneBQ_
+__RNvMs_NtNtCskkb32ezpab6_12clap_builder6parser6parserNtB4_6Parser16get_matches_with
+__RNvMs_NtNtCskkb32ezpab6_12clap_builder6parser6parserNtB4_6Parser15resolve_pending
+__RNvMs_NtNtCskkb32ezpab6_12clap_builder6parser6parserNtB4_6Parser5react
+__RNvMNtNtCskkb32ezpab6_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher6remove
+__RNvMs_NtNtCskkb32ezpab6_12clap_builder6parser6parserNtB4_6Parser16start_custom_arg
+__RNvXsc_NtNtCskkb32ezpab6_12clap_builder7builder12value_parserNtB5_15BoolValueParserNtB5_14AnyValueParser7type_idB9_
+__RNvMs0_NtNtCskkb32ezpab6_12clap_builder4util8flat_mapINtB5_5EntryNtNtB7_2id2IdNtNtNtNtB9_6parser7matches11matched_arg10MatchedArgE9or_insertB9_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtNtCskkb32ezpab6_12clap_builder6parser7matches11matched_arg10MatchedArgE8grow_oneBU_
+__RNvMNtNtNtCskkb32ezpab6_12clap_builder6parser7matches11matched_argNtB2_10MatchedArg13new_val_group
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecNtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringEE8grow_oneCskkb32ezpab6_12clap_builder
+__RNvMs_NtNtCskkb32ezpab6_12clap_builder6parser6parserNtB4_6Parser15push_arg_values
+__RNvXsC_NtNtCskkb32ezpab6_12clap_builder7builder12value_parserNtB5_15BoolValueParserNtB5_16TypedValueParser9parse_ref
+__RNvMs_NtNtCskkb32ezpab6_12clap_builder6parser6parserNtB4_6Parser12add_defaults
+__RNvXsc_NtNtCskkb32ezpab6_12clap_builder7builder12value_parserNtB5_15BoolValueParserNtB5_14AnyValueParser10parse_ref_B9_
+__RNvMNtNtCskkb32ezpab6_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher10add_val_to
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskkb32ezpab6_12clap_builder6parser6parser11ParseResultEBH_
+__RNvXsc_NtNtCskkb32ezpab6_12clap_builder7builder12value_parserINtB5_20RangedI64ValueParserhENtB5_14AnyValueParser7type_idB9_
+__RNvXsu_NtNtCskkb32ezpab6_12clap_builder7builder12value_parserINtB5_20RangedI64ValueParserhENtB5_16TypedValueParser9parse_refCsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCskkb32ezpab6_12clap_builder4util5graph10ChildGraphNtNtBG_2id2IdEEBI_
+__RNvMNtNtCskkb32ezpab6_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher10into_inner
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtCskkb32ezpab6_12clap_builder6parser6parser10PendingArgEEB13_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskkb32ezpab6_12clap_builder7builder7command7CommandECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskkb32ezpab6_12clap_builder7builder3arg3ArgECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtNtCskkb32ezpab6_12clap_builder4util2id2IdNtNtNtB1f_7builder13arg_predicate12ArgPredicateINtNtB4_6option6OptionIBC_NtNtB1W_6os_str5OsStrEEEEECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskkb32ezpab6_12clap_builder7builder3ext10ExtensionsECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCskkb32ezpab6_12clap_builder4util9any_value8AnyValueEECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCskkb32ezpab6_12clap_builder7builder7command7CommandEECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCskkb32ezpab6_12clap_builder7builder9arg_group8ArgGroupEECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskkb32ezpab6_12clap_builder7builder9arg_group8ArgGroupECsljrJvzJKKaT_11weavepy_cli
+__RNvXs2_CsljrJvzJKKaT_11weavepy_cliNtB5_3CliNtNtCskkb32ezpab6_12clap_builder6derive14FromArgMatches20from_arg_matches_mut
+__RINvMs0_NtNtNtCskkb32ezpab6_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches14try_remove_onebECsljrJvzJKKaT_11weavepy_cli
+__RINvMNtNtCskkb32ezpab6_12clap_builder4util8flat_mapINtB3_7FlatMapNtNtB5_2id2IdNtNtNtNtB7_6parser7matches11matched_arg10MatchedArgE12remove_entryeECsljrJvzJKKaT_11weavepy_cli
+__RNvMNtNtNtCskkb32ezpab6_12clap_builder6parser7matches11matched_argNtB2_10MatchedArg17into_vals_flatten
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecIBC_NtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringEEECskkb32ezpab6_12clap_builder
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringEEB1e_
+__RNvXsi_NtNtNtCs2NHf1dqD6Ob_4core4iter8adapters7flattenINtB5_13FlattenCompatINtNtNtCsdq5mISKDh0s_5alloc3vec9into_iter8IntoIterINtB1g_3VecNtNtNtCskkb32ezpab6_12clap_builder4util9any_value8AnyValueEEIB1c_B2a_EENtNtNtB9_6traits8iterator8Iterator4nextCsljrJvzJKKaT_11weavepy_cli
+__RNvXNtCs2NHf1dqD6Ob_4core3anybNtB2_3Any7type_idCskkb32ezpab6_12clap_builder
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtNtB4_4iter8adapters7flatten7FlattenINtNtNtCsdq5mISKDh0s_5alloc3vec9into_iter8IntoIterINtB1m_3VecNtNtNtCskkb32ezpab6_12clap_builder4util9any_value8AnyValueEEEECsljrJvzJKKaT_11weavepy_cli
+__RINvMNtNtCskkb32ezpab6_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs2NHf1dqD6Ob_4core6option6OptionbEECsljrJvzJKKaT_11weavepy_cli
+__RINvMs0_NtNtNtCskkb32ezpab6_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches14try_remove_onehECsljrJvzJKKaT_11weavepy_cli
+__RNvXNtCs2NHf1dqD6Ob_4core3anyhNtB2_3Any7type_idCskkb32ezpab6_12clap_builder
+__RINvMNtNtCskkb32ezpab6_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs2NHf1dqD6Ob_4core6option6OptionhEECsljrJvzJKKaT_11weavepy_cli
+__RINvMs0_NtNtNtCskkb32ezpab6_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches15try_remove_manyNtNtCsdq5mISKDh0s_5alloc6string6StringECsljrJvzJKKaT_11weavepy_cli
+__RINvMNtNtCskkb32ezpab6_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs2NHf1dqD6Ob_4core6option6OptionINtNtNtB5_7matches11arg_matches6ValuesNtNtCsdq5mISKDh0s_5alloc6string6StringEEECsljrJvzJKKaT_11weavepy_cli
+__RINvMs0_NtNtNtCskkb32ezpab6_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches14try_remove_oneNtNtCsdq5mISKDh0s_5alloc6string6StringECsljrJvzJKKaT_11weavepy_cli
+__RINvMNtNtCskkb32ezpab6_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCsdq5mISKDh0s_5alloc6string6StringEECsljrJvzJKKaT_11weavepy_cli
+__RINvMs0_NtNtNtCskkb32ezpab6_12clap_builder6parser7matches11arg_matchesNtB6_10ArgMatches14try_remove_oneNtNtCs6KxzipSu1yP_3std4path7PathBufECsljrJvzJKKaT_11weavepy_cli
+__RINvMNtNtCskkb32ezpab6_12clap_builder6parser5errorNtB3_12MatchesError6unwrapINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs6KxzipSu1yP_3std4path7PathBufEECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtNtCskkb32ezpab6_12clap_builder6parser7matches11arg_matches10ArgMatchesECsljrJvzJKKaT_11weavepy_cli
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons13raw_arg_bytes
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins16bridge_to_object
+__RNvXsz_NtCs5bNXvwCVaF0_10weavepy_vm12shared_valueNtB5_9SharedStrINtNtCs2NHf1dqD6Ob_4core7convert4FromReE4from
+__RNvXsZ_NtCsdq5mISKDh0s_5alloc6stringNtB5_6StringNtNtCs2NHf1dqD6Ob_4core3fmt5Write9write_str
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECsljrJvzJKKaT_11weavepy_cli
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsljrJvzJKKaT_11weavepy_cli
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value12SliceStoragehEE9drop_slowBL_
+__RNvCsljrJvzJKKaT_11weavepy_cli11build_flags
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBG_6string6StringEECsljrJvzJKKaT_11weavepy_cli
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VechENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneCs5bNXvwCVaF0_10weavepy_vm
+__RNvXsL_Cs5bNXvwCVaF0_10weavepy_vmNtB5_16InterpreterFlagsNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvCsljrJvzJKKaT_11weavepy_cli23run_source_with_options
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockFEuE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zECs5bNXvwCVaF0_10weavepy_vm
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockFEuE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0Cs5bNXvwCVaF0_10weavepy_vm
+__RNvMCsljrJvzJKKaT_11weavepy_cliNtB2_12EnvOverrides8from_env
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockFG_RL0_eENtNtCsllk7SLrseIa_14weavepy_parser6parser21UnicodeNameResolutionE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEB16_
+__RNvCsllk7SLrseIa_14weavepy_parser17parse_module_full
+__RNvNtCs86VDVqQvmSJ_13weavepy_lexer7scanner16tokenize_partial
+__RNvMNtCs86VDVqQvmSJ_13weavepy_lexer7scannerNtB2_7Scanner29scan_ident_or_prefixed_string
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs86VDVqQvmSJ_13weavepy_lexer5token5TokenE8grow_oneBQ_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs86VDVqQvmSJ_13weavepy_lexer
+__RNvXsc_NtNtCskkb32ezpab6_12clap_builder7builder12value_parserINtB5_20RangedI64ValueParserhENtB5_14AnyValueParser10parse_ref_CsljrJvzJKKaT_11weavepy_cli
+__RNvMs5_NtNtCskkb32ezpab6_12clap_builder7builder7commandNtB5_7Command14required_graph
+__RNvMNtNtCskkb32ezpab6_12clap_builder6parser9validatorNtB2_9Validator8validate
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCskkb32ezpab6_12clap_builder4util8flat_map7FlatMapNtNtBG_2id2IdINtNtCsdq5mISKDh0s_5alloc3vec3VecB1w_EEEBI_
+__RNvMNtNtCskkb32ezpab6_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher21fill_in_global_values
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCskkb32ezpab6_12clap_builder4util8flat_map7FlatMapNtNtBG_2id2IdNtNtNtNtBI_6parser7matches11matched_arg10MatchedArgEEBI_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs6KxzipSu1yP_3std4path7PathBufEECsljrJvzJKKaT_11weavepy_cli
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once9call_onceNCNvNtCsezj5Ays8orl_12weavepy_capi6loader27install_vm_extension_loader0E0B14_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm10ext_loader24install_extension_loader
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs5bNXvwCVaF0_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecmE8grow_oneCsgMbQNHatXAF_12aho_corasick
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13codecs_engine11utf8_encode
+__RNvMs_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13codecs_engineNtB4_6EncCtx3new
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10codecs_mod12encoding_key
+__RNvMs_NtNtCskkb32ezpab6_12clap_builder6parser6parserNtB4_6Parser19possible_subcommand
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10codecs_mod10encode_obj
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10codecs_mod10encode_str
+__RNvCsljrJvzJKKaT_11weavepy_cli8run_path
+__RNvCs5bNXvwCVaF0_10weavepy_vm25decode_source_bytes_inner
+__RNvCs5bNXvwCVaF0_10weavepy_vm25translate_source_newlines
+__RNvMNtCs86VDVqQvmSJ_13weavepy_lexer7scannerNtB2_7Scanner11scan_string
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecThjEE8grow_oneCs86VDVqQvmSJ_13weavepy_lexer
+__RNvMNtCs86VDVqQvmSJ_13weavepy_lexer7scannerNtB2_7Scanner20verify_end_of_number
+__RNvNtCsllk7SLrseIa_14weavepy_parser6parser25parse_with_flufl_tracking
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser9new_inner
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser12parse_module
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser24skip_trivia_and_newlines
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser15parse_statement
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser24check_stmt_as_ifexp_body
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser16consume_stmt_end
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCsllk7SLrseIa_14weavepy_parser3ast4StmtE8grow_oneBQ_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsllk7SLrseIa_14weavepy_parser
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser12apply_poison
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser4meta
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser6parser6ParserEBF_
+__RNvNvCscifjXFcM73a_16weavepy_compiler22collect_module_imports4walk
+__RNvNvCscifjXFcM73a_16weavepy_compiler21module_mentions_super7in_stmt
+__RINvXNvMNtCsdq5mISKDh0s_5alloc5sliceSp9to_vec_inNtNtCsllk7SLrseIa_14weavepy_parser3ast4StmtNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECscifjXFcM73a_16weavepy_compiler
+__RNvNtCscifjXFcM73a_16weavepy_compiler7ast_opt9fold_stmt
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler19compile_module_body
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler4emit
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCscifjXFcM73a_16weavepy_compiler
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler15resolve_colspan
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler12compile_stmt
+__RINvMNtCscifjXFcM73a_16weavepy_compiler6internNtB3_11InternIndex6internNtB5_8ConstantNvB3_13constant_hashEB5_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCscifjXFcM73a_16weavepy_compiler8ConstantE8grow_oneBO_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecjE8grow_oneCscifjXFcM73a_16weavepy_compiler
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCscifjXFcM73a_16weavepy_compiler9flowgraph5BlockE8grow_oneBQ_
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB4_5Block10next_instr
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECscifjXFcM73a_16weavepy_compiler
+__RNvMs0_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg13clear_visited
+__RNvNtCscifjXFcM73a_16weavepy_compiler12cpython_code14map_to_cpython
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCscifjXFcM73a_16weavepy_compiler6intern11InternIndexEBF_
+__RNvNtCscifjXFcM73a_16weavepy_compiler8validate15validate_module
+__RNvMNtCscifjXFcM73a_16weavepy_compiler8validateNtB2_5Scope3new
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCscifjXFcM73a_16weavepy_compiler8validate9ValidatorEBF_
+__RNvCscifjXFcM73a_16weavepy_compiler25collect_global_decls_deep
+__RNvCscifjXFcM73a_16weavepy_compiler19for_each_scope_expr
+__RNvXs1x_CscifjXFcM73a_16weavepy_compilerNtB6_8ConstantNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg20optimize_basic_block
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg31remove_redundant_nops_and_jumps
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg5shape
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCs2aCTYaCSZCk_8indexmap3map8IndexMapNtNtCsdq5mISKDh0s_5alloc6string6StringNtCscifjXFcM73a_16weavepy_compiler7BindingEEB1U_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCscifjXFcM73a_16weavepy_compiler9LoopFrameEEB1a_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCsdq5mISKDh0s_5alloc6string6StringuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECscifjXFcM73a_16weavepy_compiler
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtCscifjXFcM73a_16weavepy_compiler9LineIndexE9drop_slowBF_
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtNtNtCs6KxzipSu1yP_3std11collections4hash3set7HashSetNtNtB7_6string6StringEE9drop_slowCscifjXFcM73a_16weavepy_compiler
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RceE9drop_slowCscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast4StmtEECscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser3ast8StmtKindECscifjXFcM73a_16weavepy_compiler
+__RNvCs5bNXvwCVaF0_10weavepy_vm18argv_str_to_object
+__RNvXsH_NtCs5bNXvwCVaF0_10weavepy_vm12shared_valueNtB5_9SharedStrNtNtCs2NHf1dqD6Ob_4core3fmt7Display3fmt
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser22parse_subscript_single
+__RNvCscifjXFcM73a_16weavepy_compiler43should_apply_two_element_slice_optimization
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler17starunpack_helper
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler19compile_load_target
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser11parse_while
+__RNvMsC_CscifjXFcM73a_16weavepy_compilerNtB5_10CodeObject10content_eq
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast3ArgEECscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser3ast9ArgumentsECscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser3ast3ArgECscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCsllk7SLrseIa_14weavepy_parser3ast4ExprEEECscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast4ExprEECscifjXFcM73a_16weavepy_compiler
+__RNvMsH_CscifjXFcM73a_16weavepy_compilerNtB5_9LineIndex3new
+__RNvCscifjXFcM73a_16weavepy_compiler16collect_assigned
+__RNvCscifjXFcM73a_16weavepy_compiler19collect_walrus_stmt
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler19register_comp_cells
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCscifjXFcM73a_16weavepy_compiler7ColSpanE8grow_oneBO_
+__RNvCscifjXFcM73a_16weavepy_compiler21block_has_annotations
+__RNvCscifjXFcM73a_16weavepy_compiler30block_has_deferred_annotations
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler18compile_stmt_inner
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler6finish
+__RNvNtCscifjXFcM73a_16weavepy_compiler9flowgraph8optimize
+__RNvNtCscifjXFcM73a_16weavepy_compiler12cpython_code13stack_effects
+__RNvMs0_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg5chain
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg18remove_unreachable
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg20resolve_line_numbers
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCscifjXFcM73a_16weavepy_compiler8ConstantEBD_
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg27block_remove_redundant_nops
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCscifjXFcM73a_16weavepy_compiler8ConstantEEB1a_
+__RNvCscifjXFcM73a_16weavepy_compiler11make_params
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB4_9Validator10visit_stmt
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCscifjXFcM73a_16weavepy_compiler8validate5ScopeEBF_
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler3new
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCscifjXFcM73a_16weavepy_compiler12FinallyFrameEEB1a_
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg18optimize_load_fast
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCscifjXFcM73a_16weavepy_compiler9flowgraph5BlockEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionTNtNtCsdq5mISKDh0s_5alloc6string6StringBY_BY_EEECscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCscifjXFcM73a_16weavepy_compiler18DeferredAnnotationEEB1a_
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser19find_invalid_target
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser16parse_assign_rhs
+__RNvNtCsllk7SLrseIa_14weavepy_parser6parser12parse_number
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser22expect_or_forgot_comma
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser26take_trailing_type_comment
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB4_9Validator12visit_target
+__RNvCscifjXFcM73a_16weavepy_compiler20collect_target_names
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB1m_NtNtBa_3cmp10PartialOrd2ltECs5bNXvwCVaF0_10weavepy_vm
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapNtNtCsdq5mISKDh0s_5alloc6string6StringuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE12contains_keyBO_ECscifjXFcM73a_16weavepy_compiler
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCsllk7SLrseIa_14weavepy_parser3ast5CmpOpE8grow_oneBQ_
+__RNvNvCscifjXFcM73a_16weavepy_compiler21module_mentions_super7in_expr
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB4_9Validator22push_type_params_scope
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB4_9Validator14visit_function
+__RNvCscifjXFcM73a_16weavepy_compiler19collect_walrus_expr
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler25compile_default_arguments
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler18emit_make_function
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser3ast8ExprKindECscifjXFcM73a_16weavepy_compiler
+__RNvCscifjXFcM73a_16weavepy_compiler27collect_reads_assign_target
+__RNvXs1b_NtCsllk7SLrseIa_14weavepy_parser3astNtB6_8ConstantNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvXsE_CscifjXFcM73a_16weavepy_compilerNtB5_8ConstantINtNtCs2NHf1dqD6Ob_4core7convert4FromNtNtCsllk7SLrseIa_14weavepy_parser3ast8ConstantE4from
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler14unwind_fblocks
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler7emit_at
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler18close_unwind_holes
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler22super_attr_optimizable
+__RINvMsI_CscifjXFcM73a_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_expr_inners4_0EB6_
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg20const_loading_instrs
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler35import_flag_shadowed_by_module_comp
+__RINvMsI_CscifjXFcM73a_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_expr_inners2_0EB6_
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler15compile_jump_if
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler24genexp_call_optimization
+__RNCNvMsI_CscifjXFcM73a_16weavepy_compilerNtB7_8Compiler18compile_expr_inners1_0B7_
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler15compile_compare
+__RNvCscifjXFcM73a_16weavepy_compiler11emit_cmp_op
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler10patch_jump
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg36is_exit_or_eval_check_without_lineno
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser26parse_target_list_no_tuple
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser20parse_target_or_star
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser9parse_for
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCscifjXFcM73a_16weavepy_compiler9LoopFrameE8grow_oneBO_
+__RNvMs1_NtCs86VDVqQvmSJ_13weavepy_lexer5tokenNtB5_12StringPrefix5parse
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser22parse_simple_statement
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser21parse_expression_list
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser24parse_ternary_or_starred
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser13parse_ternary
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser8parse_or
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser9parse_and
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser9parse_not
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser16parse_comparison
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser12parse_bit_or
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser13parse_bit_xor
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser13parse_bit_and
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser11parse_shift
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser12parse_addsub
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser12parse_muldiv
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser11parse_unary
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser11parse_power
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser10parse_atom
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser13string_prefix
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser6lexeme
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser13decode_string
+__RNvNtCsllk7SLrseIa_14weavepy_parser6parser12strip_quotes
+__RNvNtCsllk7SLrseIa_14weavepy_parser6parser15decode_str_body
+__RNvNtCsllk7SLrseIa_14weavepy_parser6parser15cps_to_constant
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECsllk7SLrseIa_14weavepy_parser
+__RNvNtCsllk7SLrseIa_14weavepy_parser6parser16constant_str_cps
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser24check_string_expr_string
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser19at_expression_start
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser5check
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast4ExprEEB1c_
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser12parse_import
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser17parse_dotted_name
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser6expect
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser5ident
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser3eat
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser24check_import_alias_debug
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCsllk7SLrseIa_14weavepy_parser3ast5AliasE8grow_oneBQ_
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser18parse_function_def
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser26collect_pep695_type_params
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser21parse_arguments_inner
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser18try_arg_annotation
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCsllk7SLrseIa_14weavepy_parser3ast3ArgE8grow_oneBQ_
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser11parse_block
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecReE8grow_oneCsllk7SLrseIa_14weavepy_parser
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser23take_colon_type_comment
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser27take_def_body_type_comments
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser12parse_return
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser8parse_if
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser15parse_call_args
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB4_9Validator10visit_expr
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB4_9Validator13mark_assigned
+__RNvCscifjXFcM73a_16weavepy_compiler13collect_decls
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCscifjXFcM73a_16weavepy_compiler8validate5ScopeE8grow_oneBQ_
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB4_9Validator10visit_body
+__RNvXsZ_NtCsllk7SLrseIa_14weavepy_parser3astNtB5_8ExprKindNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast5AliasENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneCscifjXFcM73a_16weavepy_compiler
+__RNvXsM_NtCsllk7SLrseIa_14weavepy_parser3astNtB5_9ArgumentsNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RINvXNvMNtCsdq5mISKDh0s_5alloc5sliceSp9to_vec_inNtNtCsllk7SLrseIa_14weavepy_parser3ast3ArgNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECscifjXFcM73a_16weavepy_compiler
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast9TypeParamENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneCscifjXFcM73a_16weavepy_compiler
+__RINvXNvMNtCsdq5mISKDh0s_5alloc5sliceSp9to_vec_inNtNtCsllk7SLrseIa_14weavepy_parser3ast7KeywordNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECscifjXFcM73a_16weavepy_compiler
+__RNvNtCscifjXFcM73a_16weavepy_compiler7ast_opt9fold_expr
+__RNvMs2_NtCs2aCTYaCSZCk_8indexmap3mapINtB5_8IndexMapNtNtCsdq5mISKDh0s_5alloc6string6StringNtCscifjXFcM73a_16weavepy_compiler7BindingE11insert_fullB1r_
+__RNvMs_NtCs2aCTYaCSZCk_8indexmap5innerINtB4_4CoreNtNtCsdq5mISKDh0s_5alloc6string6StringNtCscifjXFcM73a_16weavepy_compiler7BindingE10push_entryB1o_
+__RNCNvCscifjXFcM73a_16weavepy_compiler32collect_comp_walrus_targets_stmt0B3_
+__RNvNvCscifjXFcM73a_16weavepy_compiler32collect_comp_walrus_targets_stmt4walk
+__RNCNvMsI_CscifjXFcM73a_16weavepy_compilerNtB7_8Compiler23collect_comp_cells_stmt0B7_
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler23collect_comp_cells_expr
+__RNvCscifjXFcM73a_16weavepy_compiler15clean_docstring
+__RNvXNtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB4_3VecReEINtB2_18SpecFromIterNestedB11_INtNtNtCs2NHf1dqD6Ob_4core3str4iter5SplitcEE9from_iterCscifjXFcM73a_16weavepy_compiler
+__RNvXsD_CscifjXFcM73a_16weavepy_compilerNtB5_8ConstantNtNtCs2NHf1dqD6Ob_4core3cmp9PartialEq2eq
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler15emit_store_name
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler18classify_for_store
+__RINvMs3_NtCs2aCTYaCSZCk_8indexmap3mapINtB6_8IndexMapNtNtCsdq5mISKDh0s_5alloc6string6StringNtCscifjXFcM73a_16weavepy_compiler7BindingE3geteEB1s_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneReECs5bNXvwCVaF0_10weavepy_vm
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler26compile_function_def_inner
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler26build_function_object_full
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler19child_qualname_impl
+__RNvXNtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtB6_6string6StringEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs6KxzipSu1yP_3std11collections4hash3set8IntoIterB11_EE9from_iterCscifjXFcM73a_16weavepy_compiler
+__RNvCscifjXFcM73a_16weavepy_compiler19stmt_contains_yield
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler9free_scan
+__RNvCscifjXFcM73a_16weavepy_compiler19expr_contains_yield
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler22analyze_scope_function
+__RNvCscifjXFcM73a_16weavepy_compiler21collect_reads_stmt_in
+__RNvCscifjXFcM73a_16weavepy_compiler26collect_reads_stmt_in_impl
+__RNvCscifjXFcM73a_16weavepy_compiler18collect_reads_expr
+__RNvCscifjXFcM73a_16weavepy_compiler18collect_inner_free
+__RNvCscifjXFcM73a_16weavepy_compiler23collect_inner_free_expr
+__RNvCscifjXFcM73a_16weavepy_compiler27collect_pep695_header_reads
+__RNvCscifjXFcM73a_16weavepy_compiler23collect_inner_free_impl
+__RINvMsk_NtCsdq5mISKDh0s_5alloc3vecINtB6_3VecNtNtB8_6string6StringE16extend_desugaredINtNtNtNtCs6KxzipSu1yP_3std11collections4hash3set8IntoIterBG_EECscifjXFcM73a_16weavepy_compiler
+__RINvMs3_NtCs2aCTYaCSZCk_8indexmap3mapINtB6_8IndexMapNtNtCsdq5mISKDh0s_5alloc6string6StringNtCscifjXFcM73a_16weavepy_compiler7BindingE12get_index_ofBO_EB1s_
+__RNCNvMsI_CscifjXFcM73a_16weavepy_compilerNtB7_8Compiler22analyze_scope_functions0_0B7_
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler28collect_uses_through_inlined
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler18compile_expr_inner
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler14emit_load_name
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler16var_index_or_add
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler14compile_assign
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg21scan_block_for_locals
+__RNvNtNtCskkb32ezpab6_12clap_builder6parser9validator23gather_direct_conflicts
+__RNvMs_NtNtCskkb32ezpab6_12clap_builder6parser9validatorNtB4_9Conflicts16gather_conflicts
+__RNvXsf_NtNtCs2NHf1dqD6Ob_4core5slice3cmpNtNtNtCskkb32ezpab6_12clap_builder4util2id2IdNtB5_13SliceContains14slice_containsBI_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecRNtNtNtCskkb32ezpab6_12clap_builder4util2id2IdE8grow_oneBT_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtNtCskkb32ezpab6_12clap_builder6parser7matches11matched_arg10MatchedArgECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecIBC_NtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringEEECsljrJvzJKKaT_11weavepy_cli
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser19simple_keyword_stmt
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser14parse_comp_for
+__RNvNtCscifjXFcM73a_16weavepy_compiler8validate20collect_name_targets
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB4_9Validator13record_walrus
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCsllk7SLrseIa_14weavepy_parser3ast13ComprehensionE8grow_oneBQ_
+__RNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB4_9Validator19visit_comprehension
+__RNvNtCscifjXFcM73a_16weavepy_compiler8validate22collect_walrus_targets
+__RNvNvCscifjXFcM73a_16weavepy_compiler27collect_comp_scope_walruses5visit
+__RINvXNvMNtCsdq5mISKDh0s_5alloc5sliceSp9to_vec_inNtNtCsllk7SLrseIa_14weavepy_parser3ast13ComprehensionNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECscifjXFcM73a_16weavepy_compiler
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler17compile_class_def
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler19compile_class_value
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler16build_class_body
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCscifjXFcM73a_16weavepy_compiler10ExcHandlerE8grow_oneBO_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types23install_descriptor_init
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject13set_metaclass
+__RNvCscifjXFcM73a_16weavepy_compiler25body_reads_super_or_class
+__RNCNvNCNvCscifjXFcM73a_16weavepy_compiler25body_reads_super_or_class09any_class0B7_
+__RNvCscifjXFcM73a_16weavepy_compiler19function_free_names
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler18cell_or_free_index
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCsllk7SLrseIa_14weavepy_parser3ast4ExprEECscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast9TypeParamEECscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast7KeywordEECscifjXFcM73a_16weavepy_compiler
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser15parse_class_def
+__RNCNvMs_NtCscifjXFcM73a_16weavepy_compiler8validateNtB6_9Validator10visit_stmt0B8_
+__RNvNtCscifjXFcM73a_16weavepy_compiler6mangle17mangle_class_body
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtReNtB6_7Display3fmtB8_
+__RNvMNtCscifjXFcM73a_16weavepy_compiler6mangleNtB2_7Mangler4stmt
+__RNvMNtCscifjXFcM73a_16weavepy_compiler6mangleNtB2_7Mangler4name
+__RNvMNtCscifjXFcM73a_16weavepy_compiler6mangleNtB2_7Mangler9arguments
+__RNvMNtCscifjXFcM73a_16weavepy_compiler6mangleNtB2_7Mangler4expr
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCscifjXFcM73a_16weavepy_compiler6mangle7ManglerEBF_
+__RNCNvCscifjXFcM73a_16weavepy_compiler32class_body_defs_claim_class_cell0B3_
+__RNvCscifjXFcM73a_16weavepy_compiler23method_references_class
+__RNvCscifjXFcM73a_16weavepy_compiler20stmt_needs_classdict
+__RNvCscifjXFcM73a_16weavepy_compiler23class_body_reads_dunder
+__RNvNvCscifjXFcM73a_16weavepy_compiler23class_body_reads_dunder10stmt_reads
+__RNvXs2_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7default
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins16default_builtins
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object19py_hash_bytes_slice
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm6object8HashAlgoE10initializeNCINvB2_11get_or_initNCNvBV_9hash_algo0E0zEBX_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm6object8HashAlgoE10initializeNCINvB1a_11get_or_initNCNvB1I_9hash_algo0E0zE0E0B1K_
+__RNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner17try_reserve_exactCs5bNXvwCVaF0_10weavepy_vm
+__RNvXsG_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_7DictKeyNtNtCs2NHf1dqD6Ob_4core3cmp9PartialEq2eq
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object7is_same
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object8eq_value
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object19key_needs_interp_eq
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object26instance_has_custom_dunder
+__RINvYNtNtCs5bNXvwCVaF0_10weavepy_vm8fasthash13FxBuildHasherNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtB7_6object7DictKeyEB7_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object11py_str_hash
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object9BuiltinFnE9drop_slowBK_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm6object9BuiltinFnEBF_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object13py_hash_value
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapjuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6removejECs5bNXvwCVaF0_10weavepy_vm
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRjECs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons19interactive_printer
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry16mark_default_new
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry8note_key
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry23register_text_signature
+__RNvXsq_NtCsdq5mISKDh0s_5alloc6stringNtB5_6StringNtNtCs2NHf1dqD6Ob_4core3fmt7Display3fmt
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvMsl_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_13MethodWrapper3new
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins20function_get_builtin
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types24install_gen_name_getsets
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject13new_exception
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types8exc_slot
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value7ThinArcNtNtB1j_13tuple_storage12TupleStorageEuE16get_or_init_slowNvNvNvMs1w_NtB1j_6objectNtB3c_6Object9new_tuple11EMPTY_TUPLE27___rust_std_internal_init_fnEB1j_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueSNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEBG_
+__RNvNvMNtCs5bNXvwCVaF0_10weavepy_vm13builtin_typesNtB4_12BuiltinTypes5build22install_field_defaults
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types28install_syntax_error_dunders3set
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types29install_unicode_error_dunders
+__RNvMNtCs5bNXvwCVaF0_10weavepy_vm13builtin_typesNtB2_12BuiltinTypes10as_globals
+__RNvXse_NtNtCsdq5mISKDh0s_5alloc3vec9into_iterINtB5_8IntoIterTNtNtB9_6string6StringNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEENtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4dropB1m_
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types29install_numeric_class_methods7install
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types29install_numeric_class_methods14install_method
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface7install
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins19builtin_type_dunder
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapNtNtCsdq5mISKDh0s_5alloc6string6StringINtNtB1j_6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEEuE16get_or_init_slowNvNvNvNtB3y_8builtins19builtin_type_dunder12DUNDER_CACHE27___rust_std_internal_init_fnEB3y_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14unbound_method
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_9slot_callEB4_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtCsdq5mISKDh0s_5alloc6string6StringECs5bNXvwCVaF0_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCsdq5mISKDh0s_5alloc6string6StringINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1s_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB28_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface16insert_if_absent
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface7builtin
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface21install_named_methods
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins13lookup_method
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_9str_splitEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_10str_rsplitEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_14str_splitlinesEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_14str_replace_kwEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_17str_expandtabs_kwEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_10str_encodeEB4_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface11unwrap_shim0EBH_
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object8new_list
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value7ThinArcNtNtBG_13tuple_storage12TupleStorageEEBG_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value7ThinArcINtBW_12SliceStoragehEEE10initializeNCINvB2_11get_or_initNCINvMs1w_NtBY_6objectNtB2P_6Object9new_bytesINtNtCsdq5mISKDh0s_5alloc3vec3VechEE0E0zEBY_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_15bytes_rsplit_kwEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_19bytes_splitlines_kwEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_16bytes_expandtabsEB4_
+__RINvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB7_6Object13new_bytearrayINtNtCsdq5mISKDh0s_5alloc3vec3VechEEB9_
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface23install_numeric_getsets6getset
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface25register_descriptor_kinds
+__RNvMsl_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_13MethodWrapper4func
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry6lookup
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry20is_slot_wrapper_name
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry9DescrMetaEBF_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm17builtin_docs_data15builtin_doc_for
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry6forget
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons15not_implemented
+__RNvMNtCs5bNXvwCVaF0_10weavepy_vm13builtin_typesNtB2_12BuiltinTypes5build
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5types10compute_c3
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry8register
+__RNvYNCNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry10DESCR_META0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceuE9call_onceB8_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry18set_builtin_module
+__RNvMsl_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_13MethodWrapper16wrapper_dict_for
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types20install_type_dunders9mk_getset
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types22install_module_methods7builtin
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object9new_tuple
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value7ThinArcNtNtBZ_13tuple_storage12TupleStorageEE4withNvYBU_NtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBU_EBZ_
+__RINvMNtCs5bNXvwCVaF0_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj0_EB5_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEB1c_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellINtNtB7_3vec3VecNtNtBL_6object6ObjectEEE9drop_slowBL_
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object8new_dict
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellNtNtBL_6object8DictDataEE9drop_slowBL_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm6object8DictDataEBF_
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object7new_set
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellINtNtCs2aCTYaCSZCk_8indexmap3set8IndexSetNtNtBL_6object7DictKeyNtNtBL_8fasthash13FxBuildHasherEEE9drop_slowBL_
+__RINvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB7_6Object9new_bytesINtNtCsdq5mISKDh0s_5alloc3vec3VechEEB9_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value7ThinArcINtB1J_12SliceStoragehEEE10initializeNCINvB1a_11get_or_initNCINvMs1w_NtB1L_6objectNtB3E_6Object9new_bytesINtNtCsdq5mISKDh0s_5alloc3vec3VechEE0E0zE0E0B1L_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_15bytes_decode_kwEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_12bytes_hex_kwEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_14bytes_split_kwEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_16bytes_replace_kwEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_18bytes_translate_kwEB4_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellINtNtB7_3vec3VechEEE9drop_slowBL_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14numeric_dunder
+__RNvXs1_NtNtNtCs2NHf1dqD6Ob_4core3ops8function5implsQNCNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface25register_descriptor_kinds0INtB7_5FnMutTTRNtNtBU_6object7DictKeyRNtB2h_6ObjectEEE8call_mutBU_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types30install_builtin_descriptor_get
+__RNvCs5bNXvwCVaF0_10weavepy_vm16builtin_type_doc
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry17mark_surface_only
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject14new_with_flags
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types16make_default_new
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjuEE14reserve_rehashNCINvNtB8_3map11make_hasherjuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4ZIJX5XqtQk_11weavepy_jit
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface21install_value_richcmp15richcmp_builtin
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface29install_introspection_getsets6getset
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object11new_complex
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtRNtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value9SharedStrNtB6_7Display3fmtBA_
+__RNvXsf_NtNtCs2NHf1dqD6Ob_4core5slice3cmpReNtB5_13SliceContains14slice_containsCs5bNXvwCVaF0_10weavepy_vm
+__RNvCs5bNXvwCVaF0_10weavepy_vm20builtin_slot_wrapper
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter8run_site
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14run_site_inner
+__RINvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB7_6Object18new_frozenset_fromINtNtCsdq5mISKDh0s_5alloc3vec3VecBG_EEB9_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object12FrozenSetObjE9drop_slowBK_
+__RNvMs6_NtCs5bNXvwCVaF0_10weavepy_vm5tier2NtB5_8JitState12get_compiled
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object13MethodWrapperE9drop_slowBK_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellNtNtBL_6object10PyIteratorEE9drop_slowBL_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm6object10PyIteratorEBF_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm13tuple_storage12TupleStorageE9drop_slowBK_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRTjNtNtCsdq5mISKDh0s_5alloc6string6StringEECsezj5Ays8orl_12weavepy_capi
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8class_of
+__RNvXs_NtNtNtCs2NHf1dqD6Ob_4core5array4iter10iter_innerAINtNtNtBa_3mem12maybe_uninit11MaybeUninitTNtNtCs5bNXvwCVaF0_10weavepy_vm6object7DictKeyNtB1z_6ObjectEEj1_NtB4_11PartialDrop12partial_dropB1B_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_12int_to_bytesEB4_
+__RNvMsi_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_12PyMemoryView10from_bytes
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object12PyMemoryViewE9drop_slowBK_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14run_activation
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm10specialize18init_stats_enabled
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter9quiet_run
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter10leaf_burst
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters3map3MapINtNtNtB2p_5slice4iter4IterNtCscifjXFcM73a_16weavepy_compiler8ConstantENvB17_27constant_to_object_internedEE9from_iterB17_
+__RNvCs5bNXvwCVaF0_10weavepy_vm18constant_to_object
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter4step
+__RNvMs_Cs5bNXvwCVaF0_10weavepy_vmNtB4_5Frame3pop
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter7name_at
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21globals_missing_owner
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6result6ResultNtCs5bNXvwCVaF0_10weavepy_vm11StepOutcomeNtNtBZ_5error12RuntimeErrorEEBZ_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os10build_path
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter9do_import
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys11audit_event
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os5build
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os7builtin
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os15initial_environ
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os14fsdecode_osstr
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os16stat_result_type
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject8new_user
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject14validate_bases
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject10solid_base
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os10builtin_kw
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE10initializeNCINvB2_11get_or_initNCNvNtNtBX_6stdlib2os22walk_symlinks_sentinel0E0zEBX_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvNtNtB1K_6stdlib2os22walk_symlinks_sentinel0E0zE0E0B1K_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCNvMNtBa_9lazy_lockINtB1a_8LazyLockNtNtCs5bNXvwCVaF0_10weavepy_vm8gc_trace7GcStateE5force0E0B1J_
+__RNvMs4_NtCs5bNXvwCVaF0_10weavepy_vm8gc_traceNtB5_7GcState5track
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys5build
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBG_6string6StringEECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys18version_info_value
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os15struct_seq_type
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys22disallow_instantiation
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys7builtin
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort12sort4_stableReNvYB19_NtNtBa_3cmp10PartialOrd2ltECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry15register_module
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB1A_11Interpreter15locals_fast_off0E0zEB1A_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs6dvEX9a1rCm_8lock_api5mutex5MutexNtNtCswHAcCYaqJh_11parking_lot9raw_mutex8RawMutexINtNtCs2NHf1dqD6Ob_4core6option6OptionNtCs5bNXvwCVaF0_10weavepy_vm11InterpreterEEE10initializeNCINvB1a_11get_or_initNCNvNtB3L_13vm_singletons9seed_slot0E0zE0E0B3L_
+__RNvCs5bNXvwCVaF0_10weavepy_vm22effective_ctype_locale
+__RNvMs0_NtCs5bNXvwCVaF0_10weavepy_vm3gilNtB5_8GilState3new
+__RNvMs0_NtCs5bNXvwCVaF0_10weavepy_vm3gilNtB5_8GilState7acquire
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm3gil14push_gil_guard
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache16register_builtin
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons23publish_interpreter_ptr
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache15builtin_factory
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache13frozen_source
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16push_frame_shell
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier29try_enter
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier221jit_enabled_by_config
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCs5bNXvwCVaF0_10weavepy_vm5tier28JitStateEuE16get_or_init_slowNvNvB1R_3JIT27___rust_std_internal_init_fnEB1T_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter30run_until_yield_or_return_impl
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys17venv_cfg_contents
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys13venv_cfg_path
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree11program_exe
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtB7_4path7PathBufEE10initializeNCINvB2_11get_or_initNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree7resolveE0zEB2s_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtBc_4path7PathBufEE10initializeNCINvB1a_11get_or_initNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree7resolveE0zE0E0B3g_
+__RINvMs16_NtCs6KxzipSu1yP_3std4pathNtB7_4Path4joinNtNtCsdq5mISKDh0s_5alloc6string6StringECs5bNXvwCVaF0_10weavepy_vm
+__RINvXNtNtCs6oShduW8WbL_10serde_json5value2deNtB5_5ValueNtNtCs2iBaYD3RzLF_10serde_core2de11Deserialize11deserializeQINtNtB7_2de12DeserializerNtNtB7_4read7StrReadEECs5bNXvwCVaF0_10weavepy_vm
+__RINvNvXs9_NtCs6oShduW8WbL_10serde_json2deINtB8_9MapAccesspENtNtCs2iBaYD3RzLF_10serde_core2de9MapAccess13next_key_seed12has_next_keyNtNtBa_4read7StrReadECs5bNXvwCVaF0_10weavepy_vm
+__RNvXs8_NtCs6oShduW8WbL_10serde_json4readNtB5_7StrReadNtB5_4Read9parse_str
+__RINvXs9_NtCs6oShduW8WbL_10serde_json2deINtB6_9MapAccessNtNtB8_4read7StrReadENtNtCs2iBaYD3RzLF_10serde_core2de9MapAccess15next_value_seedINtNtCs2NHf1dqD6Ob_4core6marker11PhantomDataNtNtB8_5value5ValueEECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs3_NtCs6oShduW8WbL_10serde_json2deINtB5_12DeserializerNtNtB7_4read7StrReadE13parse_integerCs5bNXvwCVaF0_10weavepy_vm
+__RNvMs3_NtCs6oShduW8WbL_10serde_json2deINtB5_12DeserializerNtNtB7_4read7StrReadE12parse_numberCs5bNXvwCVaF0_10weavepy_vm
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs6oShduW8WbL_10serde_json
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs6oShduW8WbL_10serde_json5value5ValueECs5bNXvwCVaF0_10weavepy_vm
+__RINvMNtCs2NHf1dqD6Ob_4core3stre12trim_matchesNvMNtNtB5_4char7methodsc13is_whitespaceECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8gc_trace13has_finalizer
+__RNvMs4_NtCs5bNXvwCVaF0_10weavepy_vm8gc_traceNtB5_7GcState10bump_count
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys16build_with_state
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os19struct_seq_instance
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm9hot_gates9env_flags8no_quiet0E0zE0E0B2o_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB2o_11Interpreter15locals_fast_off0E0zE0E0B2o_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCs6dvEX9a1rCm_8lock_api5mutex5MutexNtNtCswHAcCYaqJh_11parking_lot9raw_mutex8RawMutexINtNtCs2NHf1dqD6Ob_4core6option6OptionNtCs5bNXvwCVaF0_10weavepy_vm11InterpreterEEE10initializeNCINvB2_11get_or_initNCNvNtB2Y_13vm_singletons9seed_slot0E0zEB2Y_
+__RNvMs1l_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6PyFile10set_errors
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCsdq5mISKDh0s_5alloc4sync3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm3gil8GilStateEE10initializeNCINvB1a_11get_or_initNCNvB2g_14global_gil_ref0E0zE0E0B2i_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11import_path
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter8load_one
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache21is_third_party_facade
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache28find_source_shadowing_stdlib
+__RINvXsE_NtCs6KxzipSu1yP_3std4pathNtB6_7PathBufINtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits7collect12FromIteratorReE9from_iterINtNtNtBR_3str4iter5SplitcEECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache11search_dirs
+__RNvXNtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtCs6KxzipSu1yP_3std4path7PathBufEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters10filter_map9FilterMapINtNtNtB2e_5slice4iter4IterNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectENCNvMs_NtB3C_6importNtB4o_11ModuleCache11search_dirs0EE9from_iterB3C_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCsdq5mISKDh0s_5alloc3vec9into_iter8IntoIterNtNtCs6KxzipSu1yP_3std4path7PathBufEECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree11module_path
+__RINvMs16_NtCs6KxzipSu1yP_3std4pathNtB7_4Path4joinNtB7_7PathBufECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_mod15load_from_bytes
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader4take
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader8note_ref
+__RNvMNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13codecs_engineNtB2_6DecCtx3new
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object19str_from_codepoints
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13codecs_engine6DecCtxEBH_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys11intern_name
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapNtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value9SharedStruNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBR_
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object13as_bytes_view
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_mod9string_of
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_mod18tuple_to_constants
+__RNvNtCscifjXFcM73a_16weavepy_compiler12cpython_code19decode_instructions
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjjEE14reserve_rehashNCINvNtB8_3map11make_hasherjjNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECscifjXFcM73a_16weavepy_compiler
+__RNvNtCscifjXFcM73a_16weavepy_compiler12cpython_code15raw_first_instr
+__RNvMsi_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_12PyMemoryView7release
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm6object16MemoryViewBufferEBF_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_15memoryview_castEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_14memoryview_hexEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_18memoryview_tobytesEB4_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object10PyPropertyE9drop_slowBK_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_16property_init_kwEB4_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object5RangeE9drop_slowBK_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object7PySliceE9drop_slowBK_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_17staticmethod_callEB4_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB2_13str_format_kwEB4_
+__RINvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB7_6Object8from_strRNtNtCsdq5mISKDh0s_5alloc6string6StringEB9_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree7resolve
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree13landmark_walk
+__RNvMsi_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8BTreeMapNtNtBb_6string6StringNtNtCs6oShduW8WbL_10serde_json5value5ValueE6insertCs5bNXvwCVaF0_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs6oShduW8WbL_10serde_json5value5ValueE8grow_oneBQ_
+__RNvXs_NtNtCs6oShduW8WbL_10serde_json5value5indexeNtB4_5Index10index_into
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE8grow_oneBQ_
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache6insert
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8gc_trace5track
+__RNvMs4_NtCs5bNXvwCVaF0_10weavepy_vm8gc_traceNtB5_7GcState3new
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once9call_onceNCNvXs2_Cs5bNXvwCVaF0_10weavepy_vmNtB16_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_0E0B16_
+__RNvXse_NtNtCsdq5mISKDh0s_5alloc3vec9into_iterINtB5_8IntoIterNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectENtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4dropB10_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEBX_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B1K_
+__RINvMNtCs5bNXvwCVaF0_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj3_EB5_
+__RINvMs1l_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB7_6PyFile3newReBW_EB9_
+__RNvMs1k_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6IoKind9from_mode
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib14sys_monitoring5build
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5trace20monitoring_sentinels
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm5trace20monitoring_sentinels0B5_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os22struct_seq_type_layout
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable7ipnsortReNvYBT_NtNtB8_3cmp10PartialOrd2ltECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable9quicksort9quicksortReNvYB17_NtNtBa_3cmp10PartialOrd2ltECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared5pivot11median3_recReNvYB14_NtNtBa_3cmp10PartialOrd2ltECs5bNXvwCVaF0_10weavepy_vm
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm9hot_gates9env_flags8no_quiet0E0zEB1A_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm9hot_gates9env_flags8no_burst0E0zEB1A_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm9hot_gates9env_flags8no_burst0E0zE0E0B2o_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm9hot_gates9env_flags10dbg_sample0E0zEB1A_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm9hot_gates9env_flags10dbg_sample0E0zE0E0B2o_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15fork_for_thread
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17apply_run_options
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCsdq5mISKDh0s_5alloc4sync3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm3gil8GilStateEE10initializeNCINvB2_11get_or_initNCNvB1t_14global_gil_ref0E0zEB1v_
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache3get
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache19initializing_holder
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapNtNtCsdq5mISKDh0s_5alloc6string6StringyNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs5bNXvwCVaF0_10weavepy_vm
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache13live_sys_path
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6import19is_test_frozen_name
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm17frozen_code_cache3get
+__RNvXsc_NtCscifjXFcM73a_16weavepy_compiler8bytecodeNtB5_10CacheTableNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm17frozen_code_cache8get_disk
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtB7_4path7PathBufEE10initializeNCINvB2_11get_or_initNCNvNtCs5bNXvwCVaF0_10weavepy_vm17frozen_code_cache8disk_dir0E0zEB2u_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtBc_4path7PathBufEE10initializeNCINvB1a_11get_or_initNCNvNtCs5bNXvwCVaF0_10weavepy_vm17frozen_code_cache8disk_dir0E0zE0E0B3i_
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader10read_value
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader9read_byte
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader9read_code
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader8read_int
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10codecs_mod16decode_bytes_obj
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value9SharedStrEB1J_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value9SharedStruEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBU_
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader8fill_ref
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_mod8bytes_of
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_mod16tuple_of_strings
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtCscifjXFcM73a_16weavepy_compiler8ConstantEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters3map3MapINtNtNtB2o_5slice4iter4IterNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectENvB3y_25object_to_constant_publicEE9from_iterB3y_
+__RNvCs5bNXvwCVaF0_10weavepy_vm18object_to_constant
+__RNvNtCscifjXFcM73a_16weavepy_compiler12cpython_code11decode_full
+__RNvNtCscifjXFcM73a_16weavepy_compiler12cpython_code11decode_raws
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_mod13MarshalReaderEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEEB1y_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueSINtNtB4_6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEB12_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm7pycache16own_decoded_code
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm7pycache17rewrite_filenames
+__RNvMs_NtCsdq5mISKDh0s_5alloc3vecINtB4_3VechE16into_boxed_sliceCs5bNXvwCVaF0_10weavepy_vm
+__RNvXsV_CscifjXFcM73a_16weavepy_compilerNtB5_10CodeObjectNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19run_frozen_compiled
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20build_module_globals
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm7pycache14cache_path_for
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache18begin_initializing
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter10make_frame
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter23pooled_locals_from_args
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE6resizeBJ_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15recycle_scratch
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object11empty_cells
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter9run_frame
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter25run_until_yield_or_return
+__RNvCs2xvIVWRpyi2_7stacker15remaining_stack
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell4CellINtNtB1j_6option6OptionjEEzE16get_or_init_slowNvNvCs2xvIVWRpyi2_7stacker11STACK_LIMIT27___rust_std_internal_init_fnEB2A_
+__RNvCscifjXFcM73a_16weavepy_compiler20comp_clause_is_async
+__RNvCscifjXFcM73a_16weavepy_compiler19expr_contains_await
+__RNvCscifjXFcM73a_16weavepy_compiler24expr_contains_async_comp
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler21compile_comprehension
+__RNvCscifjXFcM73a_16weavepy_compiler22check_comp_walrus_nest
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCscifjXFcM73a_16weavepy_compiler15CompWalrusScopeE8grow_oneBO_
+__RNvNvCscifjXFcM73a_16weavepy_compiler25reject_walrus_in_iterable4scan
+__RNvNvCscifjXFcM73a_16weavepy_compiler22check_comp_walrus_nest5visit
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler20comp_inline_eligible
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCscifjXFcM73a_16weavepy_compiler15CompWalrusScopeEEB1a_
+__RNvCscifjXFcM73a_16weavepy_compiler17compile_comp_body
+__RNvCscifjXFcM73a_16weavepy_compiler19stamp_comp_elt_span
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler21note_identifier_const
+__RNvMs0_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg15make_load_const
+__RNvNtCscifjXFcM73a_16weavepy_compiler7ast_opt18eval_unaryop_const
+__RNvNtCscifjXFcM73a_16weavepy_compiler9flowgraph15to_ast_constant
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler9flowgraphNtB5_3Cfg11jump_thread
+__RINvMsI_CscifjXFcM73a_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_stmt_inners_0EB6_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCsllk7SLrseIa_14weavepy_parser3ast7KeywordE8grow_oneBQ_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm3gil15maybe_yield_gil
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm10ext_loader24current_extension_loader
+__RNvNtCsezj5Ays8orl_12weavepy_capi6loader17vm_load_extension
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtNtNtBW_4hash6random11RandomStateE0ECs5bNXvwCVaF0_10weavepy_vm
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler28compile_deferred_annotations
+__RNvNvCscifjXFcM73a_16weavepy_compiler24collect_self_attr_stores6target
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler13closure_index
+__RNvCscifjXFcM73a_16weavepy_compiler24collect_self_attr_stores
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10os_process8register
+__RINvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB7_6Object12new_set_fromINtNtCsdq5mISKDh0s_5alloc3vec3VecBG_EEB9_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object13identity_hash
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecyE8grow_oneCshJldXA6Gmzv_10num_bigint
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCshJldXA6Gmzv_10num_bigint
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache19builtin_static_name
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter12bind_os_path
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib9errno_mod5build
+__RNvNtNtCshJldXA6Gmzv_10num_bigint7biguint5shift12biguint_shl2
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtBJ_11Interpreter8load_one12LoadingGuardEBJ_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24lookup_global_or_builtin
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecyENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneCs5bNXvwCVaF0_10weavepy_vm
+__RNvXNtNtCshJldXA6Gmzv_10num_bigint7biguint8additionNtB4_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3AddRBN_E3add
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object15int_from_bigint
+__RINvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB6_11Interpreter17alloc_tuple_arrayKj2_EB6_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15leaf_burst_slow
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtB6_5drain5DrainB13_EE9from_iterB17_
+__RINvMsk_NtCsdq5mISKDh0s_5alloc3vecINtB6_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE14extend_trustedINtNtB6_5drain5DrainBG_EEBK_
+__RNvXs5_NtNtCsdq5mISKDh0s_5alloc3vec5drainINtB5_5DrainNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectENtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4dropBT_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11alloc_tuple
+__RINvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB6_11Interpreter17alloc_tuple_arrayKj3_EB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtBG_5boxed3BoxNtCs5bNXvwCVaF0_10weavepy_vm9InlineActEEEB1s_
+__RINvMNtCs5bNXvwCVaF0_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj2_EB5_
+__RINvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB6_11Interpreter17alloc_tuple_arrayKj1_EB6_
+__RINvMNtCs5bNXvwCVaF0_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj1_EB5_
+__RNvMsf_NtNtCs2NHf1dqD6Ob_4core3str4iterINtB5_13SplitInternalcE9next_backCs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionTNtNtCs6KxzipSu1yP_3std4path7PathBufINtNtCsdq5mISKDh0s_5alloc2rc2RcINtNtNtNtB12_11collections4hash3set7HashSetNtNtNtB12_3ffi6os_str8OsStringEEEEECsezj5Ays8orl_12weavepy_capi
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapNtNtBe_4path7PathBufTNtNtCs5bNXvwCVaF0_10weavepy_vm6import8DirStampINtNtCsdq5mISKDh0s_5alloc2rc2RcINtNtB1U_3set7HashSetNtNtNtBe_3ffi6os_str8OsStringEEEEEuE16get_or_init_slowNvNvB2S_12DIR_LISTINGS27___rust_std_internal_init_fnEB2U_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtB9_4path7PathBufECs5bNXvwCVaF0_10weavepy_vm
+__RINvXs1h_NtCs6KxzipSu1yP_3std4pathNtB7_4PathNtNtCs2NHf1dqD6Ob_4core4hash4Hash4hashNtNtNtB9_4hash6random13DefaultHasherECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6import11dir_listing
+__RNvXs2_NtNtCs6KxzipSu1yP_3std4hash6randomNtB5_13DefaultHasherNtNtCs2NHf1dqD6Ob_4core4hash6Hasher5write
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtRReNtB6_7Display3fmtCs9ZOyotO0C43_11pem_rfc7468
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapNtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringuNtNtNtBU_4hash6random11RandomStateE12contains_keyNtBQ_5OsStrECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache11find_source
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache22find_namespace_package
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20try_meta_path_import
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache16end_initializing
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter4call
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier222try_call_native_direct
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter23specialized_load_global
+__RINvMNtNtCs2NHf1dqD6Ob_4core4cell4onceINtB3_8OnceCellNtCs5bNXvwCVaF0_10weavepy_vm7LeafFnsE8try_initNCINvB2_11get_or_initNCNvMs3_BR_NtBR_11Interpreter8leaf_fns0E0zEBR_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjNtCs5bNXvwCVaF0_10weavepy_vm8LeafKindNtNtBQ_8fasthash13FxBuildHasherE6insertBQ_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6result6ResultNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectNtNtB11_5error12RuntimeErrorEEB11_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE11extend_withBJ_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier221try_call_native_bound
+__RNvXs_NtCs5bNXvwCVaF0_10weavepy_vm10specializeNtNtB6_6object8DictDataNtB4_11DictDataExt16index_of_key_str
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14str_rpartitionINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14str_rpartition
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types14make_exceptionNtNtCsdq5mISKDh0s_5alloc6string6StringEB4_
+__RNvMNtCs5bNXvwCVaF0_10weavepy_vm13builtin_typesNtB2_12BuiltinTypes7by_name
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types25make_exception_with_classNtNtCsdq5mISKDh0s_5alloc6string6StringEB4_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types19is_subclass_by_name
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object6to_str
+__RNvMsj_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10PyInstance8slot_set
+__RNvMsj_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10PyInstance17ensure_gc_tracked
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5types8SlotDataEBF_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5error18set_exception_attr
+__RNvMsj_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10PyInstance8slot_get
+__RNvMsi_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_11SlotStorage3get
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter23attach_implicit_context
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell4CellTymEEE4withNCNvNtCs5bNXvwCVaF0_10weavepy_vm5trace28monitoring_union_mask_cached0mEB1J_
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellNtNtB1j_5trace15MonitoringToolsEuE16get_or_init_slowNvNvB1Y_16MONITORING_TOOLS27___rust_std_internal_init_fnEB1j_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16handle_exception
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object20materialize_stack_at
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5error11PyExceptionEBF_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm5error14TracebackEntryEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5error14TracebackEntryEBF_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19ensure_top_py_frame
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object21materialize_stack_top
+__RNvXsf_NtCs5bNXvwCVaF0_10weavepy_vm5errorNtB5_11PyExceptionNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RINvMsk_NtCsdq5mISKDh0s_5alloc3vecINtB6_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE14extend_trustedINtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters6cloned6ClonedINtNtNtB1O_5slice4iter4IterBG_EEEBK_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15pop_frame_shell
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19recycle_frame_shell
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20recycle_frame_allocs
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCs5bNXvwCVaF0_10weavepy_vm5FrameEBD_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm5error11PyExceptionEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCs5bNXvwCVaF0_10weavepy_vm5tier216NativeActivationEEEB1A_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17call_python_owned
+__RNvMs12_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_10PyFunction4code
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16leaf_core_helper
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14leaf_core_attr
+__RNvMs9_NtCscifjXFcM73a_16weavepy_compiler8bytecodeNtB5_10CacheTable8allocate
+__RNvMs5_NtCscifjXFcM73a_16weavepy_compiler8bytecodeNtB5_9CacheSlot3new
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14leaf_core_call
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjNtCs5bNXvwCVaF0_10weavepy_vm8LeafKindEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtBT_8fasthash13FxBuildHasherE0EBT_
+__RNvCs5bNXvwCVaF0_10weavepy_vm17object_dunder_new
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18load_method_ic_hit
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21specialized_load_attr
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter9load_attr
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15load_attr_inner
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter10maybe_bind
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm10specialize28attempt_specialize_load_attr
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15try_inline_call
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13try_lean_call
+__RNvMs_NtCsdq5mISKDh0s_5alloc3vecINtB4_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE6removeBI_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13dispatch_call
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14pooled_scratch
+__RNvCs5bNXvwCVaF0_10weavepy_vm20builtin_needs_interp
+__RNvCs5bNXvwCVaF0_10weavepy_vm21code_pure_leaf_decide
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18call_from_bytecode
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter9core_call
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15call_c_profiled
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10str_result
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object11BoundMethodE9drop_slowBK_
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object9make_iter
+__RNvMsf_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE6borrowB7_
+__RNvMsf_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE10borrow_mutB7_
+__RNvMs1q_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_10PyIterator18next_value_checked
+__RNvMs1q_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_10PyIterator10next_value
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17try_lean_property
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14leaf_type_call
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtBG_6string6StringNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEEB1y_
+__RNvMs4_NtCs5bNXvwCVaF0_10weavepy_vm8gc_traceNtB5_7GcState18maybe_auto_collect
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14leaf_call_kind
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object3len
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object12str_char_len
+__RINvXNvMNtCsdq5mISKDh0s_5alloc5sliceSp9to_vec_inNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalEBO_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17try_lean_for_iter
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20specialized_for_iter
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8gc_trace16strong_count_for
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject14is_subclass_of
+__RNvMs4_NtCs5bNXvwCVaF0_10weavepy_vm8gc_traceNtB5_7GcState22rebuild_tracked_filter
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter23activate_thread_handles
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons24current_worker_thread_id
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCs6dvEX9a1rCm_8lock_api5mutex5MutexNtNtCswHAcCYaqJh_11parking_lot9raw_mutex8RawMutexINtNtNtNtB7_11collections4hash3map7HashMapyyEEE10initializeNCINvB2_11get_or_initNCNvNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons10worker_map0E0zEB3I_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib16faulthandler_mod16RegisteredThreadE8grow_oneBS_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object23collect_small_int_range
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters6cloned6ClonedINtNtNtB2p_5slice4iter4IterB13_EEE9from_iterB17_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_lenINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18new_str_split_list
+__RNvMst_NtCs5bNXvwCVaF0_10weavepy_vm12shared_valueNtB5_9SharedStr6concat
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17type_call_default
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11instantiate
+__RNvCs5bNXvwCVaF0_10weavepy_vm20object_needs_vm_iter
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16collect_iterable
+__RNvXs0_NtNtCsdq5mISKDh0s_5alloc3vec11spec_extendINtB7_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEINtB5_10SpecExtendBU_INtNtB7_9into_iter8IntoIterBU_EE11spec_extendBY_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14leaf_fused_len
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17leaf_builtin_call
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_arity
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject17metaclass_or_type
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins6b_typeINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMs11_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_10PyFunction14lean_cells_ref
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter10lean_frame
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14run_frame_lean
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19run_frame_lean_impl
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15flush_lean_cold
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15push_lean_shell
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17retire_lean_frame
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24specialized_store_subscr
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15ensure_dict_key
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15ensure_hashable
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11dict_insert
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons23current_interpreter_ptr
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object15dict_watch_bump
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object19dict_mutation_event
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6result6ResultINtNtB4_6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectENtNtB1n_5error12RuntimeErrorEEB1n_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons13ThreadHandlesEBF_
+__RNvXs0_NtCs5bNXvwCVaF0_10weavepy_vm13vm_singletonsNtB5_16InterpreterGuardNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20emit_escape_warnings
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter22run_pending_finalizers
+__RNvXsu_CskjipUxaJJsR_4itoayNtB5_8Unsigned3fmt
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCs6KxzipSu1yP_3std4path7PathBufTNtNtCs5bNXvwCVaF0_10weavepy_vm6import8DirStampINtNtCsdq5mISKDh0s_5alloc2rc2RcINtNtNtNtBU_11collections4hash3set7HashSetNtNtNtBU_3ffi6os_str8OsStringEEEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1p_NtNtNtBU_4hash6random11RandomStateE0EB1u_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs5bNXvwCVaF0_10weavepy_vm5error11PyExceptionE8grow_oneBQ_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17exception_matches
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types20instance_is_subclass
+__RNvYNCNCNvXs2_Cs5bNXvwCVaF0_10weavepy_vmNtBc_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_00INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
+__RNvYNCNCNvXs2_Cs5bNXvwCVaF0_10weavepy_vmNtBc_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_0s_0INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
+__RNvYNCNCNvXs2_Cs5bNXvwCVaF0_10weavepy_vmNtBc_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_0s1_0INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
+__RNvCs5bNXvwCVaF0_10weavepy_vm17py_frame_traverse
+__RNvXsk_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10PyInstanceNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellNtNtBL_6object6ObjectEE9drop_slowBL_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCNvMNtBa_9lazy_lockINtB1a_8LazyLockNtNtCs5bNXvwCVaF0_10weavepy_vm16weakref_registry15WeakRefRegistryE5force0E0B1J_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16pop_jump_if_step
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object9is_truthy
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter12pop_top_step
+__RNvMsi_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_11SlotStorage4iter
+__RNvYNCNCNvXs2_Cs5bNXvwCVaF0_10weavepy_vmNtBc_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_0s0_0INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
+__RNvCs5bNXvwCVaF0_10weavepy_vm21py_traceback_traverse
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm5types10PyInstanceE9drop_slowBK_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object11PyTracebackE9drop_slowBK_
+__RNvXse_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_11PyTracebackNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMs3_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_7PyFrame14refresh_locals
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object7PyFrameE9drop_slowBK_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object10FrameShellE9drop_slowBK_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object16is_function_slot
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm10specialize29attempt_specialize_store_attr
+__RNvMs3_NtCs5bNXvwCVaF0_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry3new
+__RNvCs5bNXvwCVaF0_10weavepy_vm16metaclass_method
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_dir
+__RNvMsi_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8BTreeMapNtNtBb_6string6StringNtNtB7_7set_val9SetValZSTE6insertCs5bNXvwCVaF0_10weavepy_vm
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14str_startswith
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11list_appendINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier213try_enter_osr
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13contains_full
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_setINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvCs5bNXvwCVaF0_10weavepy_vm5tsdbg0E0zEB1w_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13sort_with_key
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8dict_get
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16module_shadowing
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11import_from
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyNtNtCs5bNXvwCVaF0_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_20with_key_eq_deferredINtNtCs2NHf1dqD6Ob_4core6result6ResultINtNtB2u_6option6OptionIB32_NtBW_6ObjectEENtNtBY_5error12RuntimeErrorENCNvMs3_BY_NtBY_11Interpreter17leaf_builtin_calls1_0E0TB2p_bEEBY_
+__RINvXNtNtCsdq5mISKDh0s_5alloc3vec14spec_from_elemNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectNtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEBP_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter9make_iter
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11b_frozensetINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject17lookup_with_owner
+__RNvMs3_NtCs5bNXvwCVaF0_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry14collect_strong
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter25specialized_binary_subscr
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10b_reversed
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object9as_bigint
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15weave_frame_mod5build
+__RNvYFG_RL0_SNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEINtNtCs2NHf1dqD6Ob_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
+__RNvMs1d_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_11PyGenerator11is_finished
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16gen_method_close
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface24install_type_dict_extras0B5_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14sync_py_locals
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm16weakref_registry15queue_callbacks
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types16make_default_new0B5_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types10object_new
+__RINvNtNtCs2NHf1dqD6Ob_4core4iter8adapters11try_processINtNtB2_3map3MapQDNtNtNtB4_6traits8iterator8Iteratorp4ItemRNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEL_NCNCNvMs3_B1R_NtB1R_11Interpreter11import_star00ENtNtCsdq5mISKDh0s_5alloc6string6StringINtNtB6_6result6ResultNtNtB6_7convert10InfallibleNtNtB1R_5error12RuntimeErrorENCINvXso_B40_IB3Y_INtNtB3p_3vec3VecB3l_EB4K_EINtNtB1c_7collect12FromIteratorIB3Y_B3l_B4K_EE9from_iterBQ_E0B5v_EB1R_
+__RNvXs2J_NtNtCs2NHf1dqD6Ob_4core5slice4iterINtB6_4IterNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectENtNtNtNtBa_4iter6traits8iterator8Iterator4nextBT_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter10store_attr
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_dirINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins19str_apply_start_end
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15leaf_store_attr
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8dict_getINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real1b
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject10subclasses
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins17build_super_proxy
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17user_getattribute
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter29store_attr_instance_with_name
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real13construct_ref
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_class
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9dict_keysINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11dict_valuesINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvCs5bNXvwCVaF0_10weavepy_vm6i64_op
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecyE6resizeCshJldXA6Gmzv_10num_bigint
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins7b_range
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys12sys_getframe
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20release_finished_gen
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object13AsyncGenAwaitE9drop_slowBK_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15store_fast_step
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19leaf_load_type_attr
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins7set_addINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_classs0_0EBJ_
+__RNvMsn_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_10PyProperty4fget
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19dispatch_compare_op
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter10cmp_method
+__RNvCs5bNXvwCVaF0_10weavepy_vm8set_view
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter10obj_truthy
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19generator_send_lean
+__RNvMs5_CscifjXFcM73a_16weavepy_compilerNtB5_7JitHint17every_loop_yields
+__RNvMs3_NtCs5bNXvwCVaF0_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry5count
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14lean_call_fill
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13inline_finish
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real13ref_type_call
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18inline_gen_deliver
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter22load_method_ic_resolve
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyNtNtCs5bNXvwCVaF0_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_20with_key_eq_deferredINtNtCs2NHf1dqD6Ob_4core6result6ResultINtNtB2u_6option6OptionbENtNtBY_5error12RuntimeErrorENCNvMs3_BY_NtBY_11Interpreter17leaf_builtin_calls2_0E0TB2p_bEEBY_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11delete_attr
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter22specialized_store_attr
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15do_getattr_call
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11dict_lookup
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object12module_class
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter23module_spec_file_origin
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18is_attribute_error
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17ensure_os_environ
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real5build
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject17bump_attr_version
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10dict_items
+__RNvMsI_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_9DictWatch3new
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24alloc_plain_instance_obj
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm10specialize22type_has_attr_override
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject18setattr_is_default
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject21validate_python_bases
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object10PyFunctionE9drop_slowBK_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18deopt_call_generic
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter22specialized_compare_op
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10b_reversedINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15weave_frame_mod9frame_arg
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18maybe_donate_tuple
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectB19_EEEB1d_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object11PyGeneratorE9drop_slowBK_
+__RNvCs5bNXvwCVaF0_10weavepy_vm15make_gen_method
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15agen_init_hooks
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins20apply_set_inplace_op
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18do_issubclass_call
+__RINvMs3_NtCs2aCTYaCSZCk_8indexmap3mapINtB6_8IndexMapNtNtCs5bNXvwCVaF0_10weavepy_vm6object7DictKeyuNtNtBS_8fasthash13FxBuildHasherE12get_index_ofBO_EBS_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry24is_native_descr_accessor
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15reentrant_py_eq
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCs5bNXvwCVaF0_10weavepy_vm10QuietEntryEBD_
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm5error10type_errorNtNtCsdq5mISKDh0s_5alloc6string6StringEB4_
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object15type_name_owned
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface24install_type_dict_extras12type_prepare
+__RNvCs5bNXvwCVaF0_10weavepy_vm20mangle_class_private
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24finalize_class_namespace
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapjNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEuE16get_or_init_slowNvNvNtB2A_8builtins15DOCSTRING_CACHE27___rust_std_internal_init_fnEB2A_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15do_hasattr_call
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter23run_py_exact_with_cells
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_set
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24run_py_exact_nofree_with
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real24install_proxy_forwarding
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16try_lean_call_kw
+__RNvMsf_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBV_6string6StringEE10borrow_mutB7_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter22instantiate_plain_lean
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13instance_plan
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real19find_cached_wrapper
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins7b_rangeINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMss_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_5Range12from_bigints
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15weave_frame_mod5check
+__RNvCs5bNXvwCVaF0_10weavepy_vm15make_agen_await
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15leaf_class_attr
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm16weakref_registry11WeakRefSlotE9drop_slowBK_
+__RNvCs5bNXvwCVaF0_10weavepy_vm15is_pep604_union
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8set_copy
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types25make_exception_with_classReEB4_
+__RNvMs1_NtCs5bNXvwCVaF0_10weavepy_vm5errorNtB5_11PyException9type_name
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real16wrapper_referent
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins29class_matches_classinfo_named
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins19type_subclass_match
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtCs5bNXvwCVaF0_10weavepy_vm17InlinePlaceholderE10initializeNCINvB1a_11get_or_initNCNvB1I_18inline_placeholder0E0zE0E0B1I_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19binary_subscr_basic
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20del_type_attr_direct
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject22qualified_display_name
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys25sys_getfilesystemencoding
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter9iter_next
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10str_encodeINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1A_EEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvBG_10str_encodeE0EBI_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9slot_callINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1y_EEE4callB6_
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject13lookup_cached
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14descriptor_get
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11build_class
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter22call_metaclass_prepare
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20invoke_init_subclass
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter26check_classcell_propagated
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTyuEE14reserve_rehashNCINvNtB8_3map11make_hasheryuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs3_NtCs5bNXvwCVaF0_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry12notify_clear
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types19native_seed_for_new
+__RNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB7_11Interpreter9do_imports0_0B7_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectbEEEB1d_
+__RNvMs12_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_10PyFunction4slot
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14code_docstring
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters3map3MapINtNtNtNtB8_11collections5btree3set8IntoIterNtNtB8_6string6StringEINvMs1w_B15_B13_8from_strB3M_EEE9from_iterB17_
+__RNvMsz_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8IntoIterNtNtBb_6string6StringNtNtB7_7set_val9SetValZSTE10dying_nextCs5bNXvwCVaF0_10weavepy_vm
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14str_startswithINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins23str_match_prefix_suffix
+__RNvCs5bNXvwCVaF0_10weavepy_vm12attr_name_of
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11list_append
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object8contains
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18ensure_set_element
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14binary_op_step
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21specialized_binary_op
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18dispatch_binary_op
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvCs5bNXvwCVaF0_10weavepy_vm5tsdbg0E0zE0E0B2k_
+__RNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB7_11Interpreter18dispatch_binary_ops3_0B7_
+__RNvCs5bNXvwCVaF0_10weavepy_vm19dict_view_set_elems
+__RNvCs5bNXvwCVaF0_10weavepy_vm9binary_op
+__RNvXNtCs2aCTYaCSZCk_8indexmap3mapINtB2_8IndexMapNtNtCs5bNXvwCVaF0_10weavepy_vm6object7DictKeyuNtNtBO_8fasthash13FxBuildHasherENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBO_
+__RNvXs5_NtCsdq5mISKDh0s_5alloc5sliceSINtCs2aCTYaCSZCk_8indexmap6BucketNtNtCs5bNXvwCVaF0_10weavepy_vm6object7DictKeyuEINtB5_16SpecCloneIntoVecBy_NtNtB7_5alloc6GlobalE10clone_intoB19_
+__RNvCs5bNXvwCVaF0_10weavepy_vm10union_sets
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14do_sorted_call
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object3cmp
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15dict_setdefaultINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15dict_setdefault
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache15is_initializing
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types8exc_attr
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtRNtNtCsdq5mISKDh0s_5alloc6string6StringNtB6_7Display3fmtCs3j6f2J4rC56_13cranelift_jit
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19leaf_load_attr_recv
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15compare_op_step
+__RNvCs5bNXvwCVaF0_10weavepy_vm11compare_str
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins21construct_classmethod
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20set_type_attr_direct
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter26metaclass_descriptor_store
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18try_inline_call_kw
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18lean_call_kw_shape
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB6_3VecTNtNtB8_6string6StringNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters3zip3ZipINtNtB6_9into_iter8IntoIterB14_EIB3s_B1p_EEE9from_iterB1t_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11build_super
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter26load_attr_instance_default
+__RNvCs5bNXvwCVaF0_10weavepy_vm21callable_function_str
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter27dynamic_type_call_with_meta
+__RNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB7_11Interpreter24finalize_class_namespaces_0B7_
+__RINvMNtCs2NHf1dqD6Ob_4core3stre18trim_start_matchescECs5bNXvwCVaF0_10weavepy_vm
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10dict_itemsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvCs5bNXvwCVaF0_10weavepy_vm16is_generic_alias
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object10PyDictViewE9drop_slowBK_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm6object9DictWatchEEB11_
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters6cloned6ClonedINtNtB2l_3rev3RevINtNtNtB2p_5slice4iter4IterB13_EEEE9from_iterB17_
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object9type_name
+__RNvXsK_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_9DictWatchNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11b_frozenset
+__RNvCs5bNXvwCVaF0_10weavepy_vm11bind_method
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter8core_new
+__RNvMsj_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10PyInstance12new_deferred
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter34generic_setattr_instance_with_name
+__RNvMsN_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_8DictData20track_deferred_owner
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real16supports_weakref
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueTNtNtCsdq5mISKDh0s_5alloc6string6StringNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEECsezj5Ays8orl_12weavepy_capi
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15subscr_get_step
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface21class_getitem_builtin
+__RNvCs5bNXvwCVaF0_10weavepy_vm18make_generic_alias
+__RNvCs5bNXvwCVaF0_10weavepy_vm23collect_type_parameters
+__RNvCs5bNXvwCVaF0_10weavepy_vm15is_typevar_like
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11b_bytearrayINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14b_bytearray_kw
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15bytes_construct
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9dict_keys
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11dict_values
+__RNvCs5bNXvwCVaF0_10weavepy_vm9bignum_op
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object6as_i64
+__RNvXs2_NtNtCshJldXA6Gmzv_10num_bigint6bigint7convertNtB7_6BigIntINtNtCs2NHf1dqD6Ob_4core7convert4FromxE4from
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins19coerce_index_bigint
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys16build_with_state0B7_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17frame_locals_view
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8b_objectINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8b_object
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11call_object
+__RNvYNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types22install_object_dunders14object_setattrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types22install_object_dunders14object_setattr
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15weave_frame_mod7is_comp
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15weave_frame_mod15is_module_scope
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16invoke_finalizer
+__RNvXse_NtNtCsdq5mISKDh0s_5alloc3vec9into_iterINtB5_8IntoIterTNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectBX_EENtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4dropB11_
+__RNvXs1_NtCs5bNXvwCVaF0_10weavepy_vm5traceNtB5_20TracingSuppressGuardNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvCs5bNXvwCVaF0_10weavepy_vm24generator_frame_traverse
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object8PyModuleE9drop_slowBK_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm6object8PyModuleEBF_
+__RNvMs12_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_10PyFunction5attrs
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5types12LeafAttrKindEBF_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins7set_add
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins17set_apply_inplace
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15core_local_attr
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21leaf_fused_local_attr
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18set_membership_key
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24call_descriptor_accessor
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types20install_type_dunders12type_mro_get
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16rich_compare_obj
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21deep_equal_collection
+__RNvCs5bNXvwCVaF0_10weavepy_vm10compare_op
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8set_copyINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5error19stop_iteration_with
+__RINvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types14make_exceptionReEB4_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter26fire_caught_stop_iteration
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_classs0_0B7_
+__RNvMs0_NtCs5bNXvwCVaF0_10weavepy_vm16weakref_registryNtB5_11WeakRefSlot7upgrade
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins20generic_alias_origin
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11inline_bind
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11core_return
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14inline_deliver
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20park_suspended_boxed
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15core_gen_resume
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14try_inline_gen
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16leaf_load_method
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14core_gen_yield
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real11ref_type_eq
+__RNvMsh_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellINtNtCs2NHf1dqD6Ob_4core6option6OptionReEE3getB7_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvCs5bNXvwCVaF0_10weavepy_vm14fusion_enabled0E0zEB1w_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvCs5bNXvwCVaF0_10weavepy_vm14fusion_enabled0E0zE0E0B2k_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15load_fast_value
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5error21attribute_error_named
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21do_list_or_tuple_call
+__RNCINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB4_10str_encodeEs_0B6_
+__RNCINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB4_9slot_callEs_0B6_
+__RNvMs4_NtCs5bNXvwCVaF0_10weavepy_vm8gc_traceNtB5_7GcState12collect_impl
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons24drain_pending_cext_drops
+__RNvMs4_NtCs5bNXvwCVaF0_10weavepy_vm8gc_traceNtB5_7GcState18collect_generation
+__RNvCs5bNXvwCVaF0_10weavepy_vm13iter_traverse
+__RNvMs1q_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_10PyIterator12gc_referents
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8gc_trace19clear_object_fields
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7io_full5build
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io5build
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io19build_iobase_family
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io13iobase_method
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io19install_fileio_ctor0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io13make_buffered
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io13make_buffered0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io23install_delegating_attr
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io18make_memory_stream
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io18make_memory_stream0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io15set_type_module
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io32make_incremental_newline_decoder
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io32make_incremental_newline_decoder0B7_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2io8IoFamilyEBH_
+__RNCNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os22struct_seq_type_layout0se_0B9_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os9os_fspath
+__RNCINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB4_9str_splitEs_0B6_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_splitINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1y_EEE4callB6_
+__RNvMsf_NtNtCs2NHf1dqD6Ob_4core3str4iterINtB5_13SplitInternalReE4nextCs5bNXvwCVaF0_10weavepy_vm
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8str_joinINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8str_join
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18join_exact_strings
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15subscr_set_step
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter12store_subscr
+__RNvCs5bNXvwCVaF0_10weavepy_vm22apply_slice_assignment
+__RNvCs5bNXvwCVaF0_10weavepy_vm17slice_bound_index
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCsdq5mISKDh0s_5alloc3vec6splice6SpliceINtNtBG_9into_iter8IntoIterNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEEB1P_
+__RNvXs1_NtNtCsdq5mISKDh0s_5alloc3vec6spliceINtB5_6SpliceINtNtB7_9into_iter8IntoIterNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEENtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4dropB1m_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20leaf_instance_dunder
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter25default_getattribute_slow
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types26make_exception_with_object
+__RNvMs1s_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object4repr
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object16bytes_repr_inner
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24build_exception_instance
+__RNvCs5bNXvwCVaF0_10weavepy_vm21lookup_exception_init
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24instantiate_raised_class
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19normalize_exception
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter28break_implicit_context_cycle
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14sync_exc_attrs
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCs5bNXvwCVaF0_10weavepy_vm5error11PyExceptionEEB1e_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm6object11PyTracebackEBF_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_rfindINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_rfind
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins19char_offset_to_byte
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins19byte_offset_to_char
+__RNvMst_NtCs5bNXvwCVaF0_10weavepy_vm12shared_valueNtB5_9SharedStr6repeat
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10str_rstripINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10str_rstrip
+__RNvXNtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB4_3VeccEINtB2_18SpecFromIterNestedcNtNtNtCs2NHf1dqD6Ob_4core3str4iter5CharsE9from_iterCs5bNXvwCVaF0_10weavepy_vm
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins12str_endswithINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins12str_endswith
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter12do_next_call
+__RNvYFG0_RL1_SNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectRL0_STNtNtCsdq5mISKDh0s_5alloc6string6StringBb_EEINtNtCs2NHf1dqD6Ob_4core6result6ResultBb_NtNtBf_5error12RuntimeErrorEINtNtNtB1L_3ops8function2FnTRBa_RBX_EE4callBf_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os10os_stat_kw
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os10dir_fd_arg
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os14path_to_string
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8str_findINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvXsv_NtNtCs2NHf1dqD6Ob_4core3str7patternNtB5_11StrSearcherNtB5_8Searcher10next_match
+__RNCINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB4_15bytes_decode_kwEs_0B6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_lower
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os15stat_seq_finish
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter7repr_of
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11list_extendINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMsc_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10TypeObject6lookup
+__RNvCs5bNXvwCVaF0_10weavepy_vm17resolve_metaclass
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21invoke_set_name_hooks
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11import_star
+__RNvXs2J_NtNtCs2NHf1dqD6Ob_4core5slice4iterINtB6_4IterNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectENtNtNtNtBa_4iter6traits8iterator8Iterator9size_hintBT_
+__RNvMs12_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_10PyFunction8set_slot
+__RNvCs5bNXvwCVaF0_10weavepy_vm15instance_method
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5error21io_error_to_py_named2
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5error24set_oserror_filename_obj
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter29leaf_resolve_instance_attr_ix
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCs5bNXvwCVaF0_10weavepy_vm8LeafAttrEBD_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8str_find
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15bytes_decode_kwINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1F_EEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10bytes_data
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os21stat_result_from_meta
+__RNvYNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types22install_object_dunders11object_initINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types22install_object_dunders11object_init
+__RNvMsi_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_11SlotStorage8index_of
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19run_py_exact_nofree
+__RNvMsj_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_10PyInstance11try_recycle
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14load_attr_type
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry15is_surface_only
+__RNvXsM_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_8DictDataNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14core_pure_call
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_lowerINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11list_extend
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18is_data_descriptor
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os10os_listdir
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os9os_getgid
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter12format_value
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21core_pure_global_call
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib2os9os_getuid
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10signal_mod31register_current_thread_pthread
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8dict_popINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12warnings_mod15default_filters
+__RNvMs3_NtCs9TUm298FobZ_17cranelift_codegen3isaDNtB5_9TargetIsaEL_15frontend_config
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCs9TUm298FobZ_17cranelift_codegen3isa10IsaBuilderINtNtB4_6result6ResultINtNtCsdq5mISKDh0s_5alloc4sync3ArcDNtBE_9TargetIsaEL_ENtNtBG_6result12CodegenErrorEEECs4ZIJX5XqtQk_11weavepy_jit
+__RNvMsi_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8BTreeMapjNtNtB7_7set_val9SetValZSTE6insertCs4ZIJX5XqtQk_11weavepy_jit
+__RNvMs4_NtNtCsdq5mISKDh0s_5alloc11collections9vec_dequeINtB5_8VecDequejE13push_back_mutCs4ZIJX5XqtQk_11weavepy_jit
+__RINvXNtNtCsdq5mISKDh0s_5alloc3vec14spec_from_elemINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs4ZIJX5XqtQk_11weavepy_jit5value7JitTypeENtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEB1r_
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze11infer_block
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapjmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE12contains_keyjECs4ZIJX5XqtQk_11weavepy_jit
+__RNCNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212warm_compile0s2_0B7_
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze10emit_block
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14str_replace_kwINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1E_EEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm10specialize26attempt_specialize_call_kw
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real26instance_method_trampoline
+__RNvXs1_NtCs9TUm298FobZ_17cranelift_codegen3isaINtB5_10IsaBuilderINtNtCs2NHf1dqD6Ob_4core6result6ResultINtNtCsdq5mISKDh0s_5alloc4sync3ArcDNtB5_9TargetIsaEL_ENtNtB7_6result12CodegenErrorEENtNtB7_8settings12Configurable6enableCs4ZbJrwhxE8U_16cranelift_native
+__RNvCs4ZbJrwhxE8U_16cranelift_native7builder
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze2SEE8grow_oneBQ_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7imp_mod5build
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7imp_mod7builtin
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins22construct_staticmethod
+__RNvMs_Cs5bNXvwCVaF0_10weavepy_vmNtB4_5Frame3top
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18do_isinstance_call
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21deopt_load_attr_taken
+__RNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB7_11Interpreter16module_shadowings0_0B7_
+__RNCINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB4_14str_replace_kwEs_0B6_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7imp_mod14imp_is_builtin
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm10specialize26kw_uncovered_have_defaults
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real1b
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real26instance_dunder_trampoline
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real14module_getattr
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11dict_remove
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19kw_names_bind_check
+__RNvXs1_NtCs9TUm298FobZ_17cranelift_codegen8settingsNtB5_7BuilderNtB5_12Configurable3set
+__RNvMNtCs9TUm298FobZ_17cranelift_codegen7contextNtB2_7Context3new
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze12analyze_impl
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze12analyze_once
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze12plan_rewrite
+__RNvXNtCseIGkHRfguhS_9hashbrown3mapINtB2_7HashMapjuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneCs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze12build_blocks
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapjuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE12contains_keyjECs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze16reachable_blocks
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze14emit_attr_lane
+__RNvXNtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB4_3VecmEINtB2_18SpecFromIterNestedmINtNtNtNtCs6KxzipSu1yP_3std11collections4hash3set8IntoItermEE9from_iterCs4ZIJX5XqtQk_11weavepy_jit
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionIBC_NtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze2SEEEEEB1C_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins23reentrant_load_attr_opt
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15property_setterINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins16property_payload
+__RNvMsn_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_10PyProperty4fdel
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14wstr_attr_dict
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter18ensure_module_spec
+__RNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB7_11Interpreter16bind_python_argss8_0B7_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter27specialized_unpack_sequence
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter25recursive_isinstance_type
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7imp_mod13imp_is_frozen
+__RINvMs3_NtCsdq5mISKDh0s_5alloc3stre7replaceReECs5bNXvwCVaF0_10weavepy_vm
+__RNCNvMNtCs5bNXvwCVaF0_10weavepy_vm13builtin_typesNtB4_12BuiltinTypes5builds4_0B6_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7imp_mod18imp_create_builtin
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real5build
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexINtNtNtNtB7_11collections4hash3map7HashMapyjEEE10initializeNCINvB2_11get_or_initNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10signal_mod16pthread_registry0E0zEB2M_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTyjEE14reserve_rehashNCINvNtB8_3map11make_hasheryjNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8dict_pop
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20kw_names_fill_locals
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12warnings_mod5build
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12warnings_mod10builtin_kw
+__RNvYNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12warnings_mod5builds_0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceTRSNtNtBa_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1R_EEE9call_onceBa_
+__RNvMs3_NtCs4ZIJX5XqtQk_11weavepy_jit6engineNtB5_9JitEngine3new
+__RNvMs0_NtCs9TUm298FobZ_17cranelift_codegen8settingsNtB5_7Builder6lookup
+__RNvMsn_NtCs9TUm298FobZ_17cranelift_codegen8settingsNtB5_5Flags3new
+__RNvMNtCs3j6f2J4rC56_13cranelift_jit7backendNtB2_10JITBuilder8with_isa
+__RNvMs0_NtCs3j6f2J4rC56_13cranelift_jit7backendNtB5_9JITModule3new
+__RNvMs3_NtCs9TUm298FobZ_17cranelift_codegen3isaDNtB5_9TargetIsaEL_17default_call_conv
+__RNvMNtCs6QZsINA9Usy_18cranelift_frontend8frontendNtB2_22FunctionBuilderContext3new
+__RNvXs3_NtNtCs2NHf1dqD6Ob_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze14compute_livein
+__RNvXNtCseIGkHRfguhS_9hashbrown3mapINtB2_7HashMapmuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneCs4ZIJX5XqtQk_11weavepy_jit
+__RNvXsL_NtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB5_9CtorStateNtNtCs2NHf1dqD6Ob_4core7default7Default7default
+__RINvXNtNtCsdq5mISKDh0s_5alloc3vec14spec_from_elemINtNtCs2NHf1dqD6Ob_4core6option6OptionINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze2SEEENtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEB1B_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze2SEENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze13step_abstract
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze15infer_load_attr
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB2_9PathArena5names
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier220attr_fingerprint_obj
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBG_6string6StringEECs4ZIJX5XqtQk_11weavepy_jit
+__RINvXNtNtCsdq5mISKDh0s_5alloc3vec14spec_from_elemINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir6TBlockENtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEB1r_
+__RNvXNtNtNtCsdq5mISKDh0s_5alloc11collections9vec_deque14spec_from_iterINtB4_8VecDequejEINtB2_12SpecFromIterjINtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters6copied6CopiedINtNtNtB1Q_5slice4iter4IterjEEE14spec_from_iterCs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze10emit_instr
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir5TStmtE8grow_oneBQ_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir12AttrSiteMetaE8grow_oneBQ_
+__RNvMs4_NtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB5_5ESlot8is_plain
+__RNvXNtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB4_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir13CompSavedMetaEINtB2_18SpecFromIterNestedB11_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters6copied6CopiedINtNtNtNtCs6KxzipSu1yP_3std11collections4hash3map6ValuesmB11_EEE9from_iterB15_
+__RINvMs_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecmE8dedup_byNCNvMs5_B5_Bv_5dedup0ECs4ZIJX5XqtQk_11weavepy_jit
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze9CtorStateEBF_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree8contains
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18construct_property
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins21property_init_members
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins13property_with
+__RNvMsn_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_10PyProperty4fset
+__RNvMsn_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_10PyProperty3doc
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19do_dict_update_call
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15dict_merge_from
+__RNvYNtNtCs3j6f2J4rC56_13cranelift_jit7backend9JITModuleNtNtCs7oIyYg99ywV_16cranelift_module6module6Module13clear_contextCs4ZIJX5XqtQk_11weavepy_jit
+__RNvMsz_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8IntoIterNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities4InstINtCsbLrsMrHhqlD_8smallvec8SmallVecANtNtB1b_15user_stack_maps17UserStackMapEntryj4_EE10dying_nextB1d_
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs6QZsINA9Usy_18cranelift_frontend
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs6QZsINA9Usy_18cranelift_frontend
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir3dfg15ValueDataPackedE8grow_oneBS_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCs9TUm298FobZ_17cranelift_codegen8machinst16CompiledCodeBaseNtNtB10_6buffer5FinalEEEB12_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs9TUm298FobZ_17cranelift_codegen
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockNtNtCs6QZsINA9Usy_18cranelift_frontend3ssa12SSABlockDataE20resize_for_index_mutB1W_
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs9TUm298FobZ_17cranelift_codegen
+__RNvXs0_NtCs6QZsINA9Usy_18cranelift_frontend8frontendNtB5_15FuncInstBuilderNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir7builder15InstBuilderBase5build
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockNtNtB11_6layout9BlockNodeE20resize_for_index_mutB13_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockNtNtCs6QZsINA9Usy_18cranelift_frontend8frontend11BlockStatusE20resize_for_index_mutB1W_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir12instructions15InstructionDataE8grow_oneBS_
+__RNvMsb_NtNtCs9TUm298FobZ_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph17make_inst_results
+__RNvMs4_NtNtCs9TUm298FobZ_17cranelift_codegen2ir6layoutNtB5_6Layout11append_inst
+__RNvMs3_NtCsjh7C7x83D1_16cranelift_entity4listINtB5_10EntityListNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueE8as_sliceB15_
+__RNvMsb_NtNtCs9TUm298FobZ_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph35non_tail_call_or_try_call_signature
+__RNvMsc_NtNtCs9TUm298FobZ_17cranelift_codegen2ir12instructionsNtB5_15InstructionData12analyze_call
+__RNvXNvMsb_NtNtCs9TUm298FobZ_17cranelift_codegen2ir3dfgNtB8_13DataFlowGraph17inst_result_typesNtB2_15InstResultTypesNtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits8iterator8Iterator4next
+__RNvMsf_NtNtCs9TUm298FobZ_17cranelift_codegen2ir12instructionsNtB5_17OperandConstraint7resolve
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities4InstNtNtB11_6layout8InstNodeE20resize_for_index_mutB13_
+__RNvMs0_NtNtCs9TUm298FobZ_17cranelift_codegen2ir6layoutNtB5_6Layout15assign_inst_seq
+__RNvXNtNtCs9TUm298FobZ_17cranelift_codegen2ir3dfgNtB2_5InstsINtNtNtCs2NHf1dqD6Ob_4core3ops5index5IndexNtNtB4_8entities4InstE5index
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir5types4TypeE8grow_oneBS_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtCs6QZsINA9Usy_18cranelift_frontend8variable8VariableIBE_NtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockINtNtB4_13packed_option12PackedOptionNtB1X_5ValueEEE20resize_for_index_mutB11_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockINtNtB4_13packed_option12PackedOptionNtBZ_5ValueEE20resize_for_index_mutCs6QZsINA9Usy_18cranelift_frontend
+__RNvMs3_NtCsjh7C7x83D1_16cranelift_entity4listINtB5_10EntityListNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueE4pushCs4ZIJX5XqtQk_11weavepy_jit
+__RNvMs3_NtCsjh7C7x83D1_16cranelift_entity4listINtB5_10EntityListNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueE4growCs4ZIJX5XqtQk_11weavepy_jit
+__RNvMNtNtCs9TUm298FobZ_17cranelift_codegen2ir12instructionsNtB2_9BlockCall5block
+__RNvMs1_NtCs6QZsINA9Usy_18cranelift_frontend3ssaNtB5_10SSABuilder25declare_block_predecessor
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser16parse_dict_value
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser17expect_dict_colon
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCsllk7SLrseIa_14weavepy_parser3ast4ExprEE8grow_oneB1s_
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler15compile_subdict
+__RNvCscifjXFcM73a_16weavepy_compiler25expr_contains_typeparamfn
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser11parse_yield
+__RNvCscifjXFcM73a_16weavepy_compiler31collect_inlined_comp_inner_free
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler23compile_slice_two_parts
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapymNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCscifjXFcM73a_16weavepy_compiler
+__RINvNvNtCscifjXFcM73a_16weavepy_compiler6intern13constant_hash4hashNtNtNtCs6KxzipSu1yP_3std4hash6random13DefaultHasherEB6_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTymEE14reserve_rehashNCINvNtB8_3map11make_hasherymNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECscifjXFcM73a_16weavepy_compiler
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtCs2NHf1dqD6Ob_4core6option6OptionmEE8grow_oneCscifjXFcM73a_16weavepy_compiler
+__RNvCscifjXFcM73a_16weavepy_compiler18collect_comp_reads
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapNtNtCsdq5mISKDh0s_5alloc6string6StringuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6removeBO_ECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast13ComprehensionEECscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser3ast13ComprehensionECscifjXFcM73a_16weavepy_compiler
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler29collect_global_walrus_shallow
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler21collect_names_shallow
+__RNvCscifjXFcM73a_16weavepy_compiler12comp_inlines
+__RNvCscifjXFcM73a_16weavepy_compiler20expr_yields_in_scope
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler14comp_own_cells
+__RNvCscifjXFcM73a_16weavepy_compiler29collect_comp_scope_inner_free
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler12comp_symbols
+__RNvCscifjXFcM73a_16weavepy_compiler28collect_target_names_ordered
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler27collect_nested_comp_symbols
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmNtNtCscifjXFcM73a_16weavepy_compiler8bytecode6OpCodeNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBS_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmNtNtCscifjXFcM73a_16weavepy_compiler8bytecode6OpCodeEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvMNtCs86VDVqQvmSJ_13weavepy_lexer7scannerNtB2_7Scanner19scan_fstring_extent
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVeccE8grow_oneCs86VDVqQvmSJ_13weavepy_lexer
+__RNvMNtCs86VDVqQvmSJ_13weavepy_lexer7scannerNtB2_7Scanner25scan_fstring_field_extent
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser17fstring_parts_for
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser24parse_fstring_body_inner
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser19parse_fstring_field
+__RNvNtCsllk7SLrseIa_14weavepy_parser6parser28scrub_fstring_field_comments
+__RINvMNtCs2NHf1dqD6Ob_4core3stre12trim_matchesNCNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtBM_6Parser19parse_fstring_field0EBO_
+__RNvNtCs86VDVqQvmSJ_13weavepy_lexer7scanner8tokenize
+__RNvNtCsllk7SLrseIa_14weavepy_parser6parser16remap_expr_spans
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser17parse_import_from
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser3ast7KeywordECscifjXFcM73a_16weavepy_compiler
+__RINvMsI_CscifjXFcM73a_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_expr_inners3_0EB6_
+__RNvNtCscifjXFcM73a_16weavepy_compiler7ast_opt10eval_binop
+__RNvNtCscifjXFcM73a_16weavepy_compiler7ast_opt6int_of
+__RNvNtCscifjXFcM73a_16weavepy_compiler7ast_opt10str_points
+__RNvMNtCsdq5mISKDh0s_5alloc5sliceSm6repeatCscifjXFcM73a_16weavepy_compiler
+__RNvNtCscifjXFcM73a_16weavepy_compiler7ast_opt8make_str
+__RNvNtCscifjXFcM73a_16weavepy_compiler7ast_opt15optimize_format
+__RNvXNtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB4_3VeccEINtB2_18SpecFromIterNestedcNtNtNtCs2NHf1dqD6Ob_4core3str4iter5CharsE9from_iterCscifjXFcM73a_16weavepy_compiler
+__RNvCscifjXFcM73a_16weavepy_compiler14constant_slice
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueTNtCscifjXFcM73a_16weavepy_compiler8ConstantBC_BC_EEBE_
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftTjjENvYB1m_NtNtBa_3cmp10PartialOrd2ltECscifjXFcM73a_16weavepy_compiler
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser11parse_raise
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser16parse_else_block
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler15emit_super_attr
+__RNvNtCscifjXFcM73a_16weavepy_compiler6intern9name_hash
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree18write_install_json
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree18write_install_json0B5_
+__RINvXNtNtCs6oShduW8WbL_10serde_json5value3serNtB5_5ValueNtNtCs2iBaYD3RzLF_10serde_core3ser9Serialize9serializeQINtNtB7_3ser10SerializerQINtNtCsdq5mISKDh0s_5alloc3vec3VechENtB1O_15PrettyFormatterEECs5bNXvwCVaF0_10weavepy_vm
+__RNvXs1_NtCs6oShduW8WbL_10serde_json3serQINtB5_10SerializerQINtNtCsdq5mISKDh0s_5alloc3vec3VechENtB5_15PrettyFormatterENtNtCs2iBaYD3RzLF_10serde_core3ser10Serializer13serialize_strCs5bNXvwCVaF0_10weavepy_vm
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree12write_atomic
+__RINvNvXs0_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB8_8BTreeMappppENtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone13clone_subtreeNtNtBe_6string6StringNtNtCs6oShduW8WbL_10serde_json5value5ValueNtNtBe_5alloc6GlobalECs5bNXvwCVaF0_10weavepy_vm
+__RNvXs_NtNtCs6oShduW8WbL_10serde_json5value5indexeNtB4_5Index15index_or_insert
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm11stdlib_tree12uname_fields5field
+__RINvMs3_NtCsdq5mISKDh0s_5alloc3stre7replacecECs5bNXvwCVaF0_10weavepy_vm
+__RNvYNCNvMNtCs86VDVqQvmSJ_13weavepy_lexer7scannerNtB7_7Scanner11scan_number0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceThEE9call_onceB9_
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser9parse_del
+__RNvMsI_CscifjXFcM73a_16weavepy_compilerNtB5_8Compiler14compile_delete
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14format_obj_str
+__RNvCs5bNXvwCVaF0_10weavepy_vm20format_via_spec_impl
+__RINvNtCsdq5mISKDh0s_5alloc3str17join_generic_copyehReECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter30register_source_with_linecache
+__RNCINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB4_14str_splitlinesEs_0B6_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14str_splitlinesINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1E_EEE4callB6_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8list_popINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins8list_pop
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins19code_synthetic_attr
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEINtB4_18SpecFromIterNestedB13_INtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters3map3MapINtNtB2l_6cloned6ClonedINtNtNtB2p_5slice4iter4IterNtCscifjXFcM73a_16weavepy_compiler8ConstantEENvB17_25constant_to_object_publicEE9from_iterB17_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtCscifjXFcM73a_16weavepy_compiler10CodeObjectE9drop_slowBI_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCscifjXFcM73a_16weavepy_compiler5VmExtECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtCscifjXFcM73a_16weavepy_compiler13WireOverridesEEEB1y_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24run_interpreter_shutdown
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10atexit_mod17run_exit_handlers
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib16interpreters_mod8RegistryEE10initializeNCINvB2_11get_or_initNCNvB1o_8registry0E0zEB1s_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtNtBa_6poison5mutex5MutexNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib16interpreters_mod8RegistryEE10initializeNCINvB1a_11get_or_initNCNvB2b_8registry0E0zE0E0B2f_
+__RNvXNtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB4_3VecyEINtB2_18SpecFromIterNestedyINtNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters6copied6CopiedINtNtNtNtCs6KxzipSu1yP_3std11collections4hash3map4KeysyNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib16interpreters_mod16InterpreterEntryEEE9from_iterB3n_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtB1a_5error12RuntimeErrorEEEB1a_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB1a_6object6ObjectEEEB1a_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtB1a_6option6OptionINtNtCsdq5mISKDh0s_5alloc5boxed3BoxSNtNtNtCs5bNXvwCVaF0_10weavepy_vm5types10type_cache5EntryEEEEB2J_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter23run_shutdown_finalizers
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13flush_streams
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16current_sys_attr
+__RNvMs1l_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6PyFile9is_closed
+__RNvMsf_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtBV_6string6StringNtNtB7_6object6ObjectEEE6borrowB7_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15file_check_open
+__RNvMs1l_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6PyFile15flush_write_buf
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCs5bNXvwCVaF0_10weavepy_vm16InterpreterFlagsECsljrJvzJKKaT_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCsljrJvzJKKaT_11weavepy_cli12EnvOverridesEBD_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCsljrJvzJKKaT_11weavepy_cli3CliEBD_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtB1a_6object6ObjectB2l_EEEEB1a_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm3gil8GilGuardEEEB2h_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyNtNtCs5bNXvwCVaF0_10weavepy_vm6object18KeyComparisonStateEB19_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCs5bNXvwCVaF0_10weavepy_vm5tier28JitStateEEB1J_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCs5bNXvwCVaF0_10weavepy_vm16CodeConstObjectsEBD_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCscifjXFcM73a_16weavepy_compiler8ConstantEECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCscifjXFcM73a_16weavepy_compiler8ConstantECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtCscifjXFcM73a_16weavepy_compiler13WireOverridesEEECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB1a_13vm_singletons13ThreadHandlesEEEB1a_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib16faulthandler_mod16RegisteredThreadEBH_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapNtNtCsdq5mISKDh0s_5alloc6string6StringINtNtB19_6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEEEB3o_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value7ThinArcNtNtB19_13tuple_storage12TupleStorageEEB19_
+__RNvXsh_Cs5bNXvwCVaF0_10weavepy_vmNtB5_9LeanFrameNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMs1l_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6PyFile14get_extra_attr
+__RNvMs1l_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6PyFile5flush
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtCs5bNXvwCVaF0_10weavepy_vm6object7DictKeyNtB1b_6ObjectEEEB1d_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtB1a_6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons23FaulthandlerThreadGuardEEEB27_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10file_flushINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10file_flush
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib16faulthandler_mod16note_thread_exit
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtNtCs6KxzipSu1yP_3std6thread9lifecycle6PacketlEE9drop_slowCsljrJvzJKKaT_11weavepy_cli
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellNtNtB1a_6object14ReprGuardStateEEB1a_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtCsdq5mISKDh0s_5alloc3vec3VecTjNtNtCs5bNXvwCVaF0_10weavepy_vm8builtins12SetInplaceOpEEEEB2j_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellTNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectB1H_EEEB1L_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapjNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEEB2q_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellNtNtB19_5trace15MonitoringToolsEEB19_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellINtNtNtNtBa_11collections4hash3set7HashSetNtNtCsdq5mISKDh0s_5alloc6string6StringEEEB19_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapNtNtBa_4path7PathBufTNtNtCs5bNXvwCVaF0_10weavepy_vm6import8DirStampINtNtCsdq5mISKDh0s_5alloc2rc2RcINtNtB1K_3set7HashSetNtNtNtBa_3ffi6os_str8OsStringEEEEEEB2K_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueSNtNtCs5bNXvwCVaF0_10weavepy_vm6object7DictKeyEBG_
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtNtNtCs6KxzipSu1yP_3std11collections4hash3set7HashSetNtNtNtBM_3ffi6os_str8OsStringEE9drop_slowCs5bNXvwCVaF0_10weavepy_vm
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer5guard
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer9emit_exit
+__RNvMs2_NtCsjh7C7x83D1_16cranelift_entity4listINtB5_8ListPoolNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueE7reallocCs4ZIJX5XqtQk_11weavepy_jit
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs4ZIJX5XqtQk_11weavepy_jit
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer10emit_block
+__RNvMs1_NtCs6QZsINA9Usy_18cranelift_frontend3ssaNtB5_10SSABuilder16use_var_nonlocal
+__RNvMs1_NtCs6QZsINA9Usy_18cranelift_frontend3ssaNtB5_10SSABuilder17run_state_machine
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer15list_helper_sig
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir6TBlockEBF_
+__RNvMs4_NtCs6QZsINA9Usy_18cranelift_frontend8frontendNtB5_15FunctionBuilder15seal_all_blocks
+__RNvMs1_NtCs6QZsINA9Usy_18cranelift_frontend3ssaNtB5_10SSABuilder25begin_predecessors_lookup
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs6QZsINA9Usy_18cranelift_frontend3ssa4CallE8grow_oneBQ_
+__RNvMsc_NtNtCs9TUm298FobZ_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph18remove_block_param
+__RNvMs2_NtCsjh7C7x83D1_16cranelift_entity4listINtB5_8ListPoolNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueE4freeB12_
+__RNvMs2_NtCsjh7C7x83D1_16cranelift_entity4listINtB5_8ListPoolNtNtCs6QZsINA9Usy_18cranelift_frontend8variable8VariableE4freeB10_
+__RNvMs4_NtCs6QZsINA9Usy_18cranelift_frontend8frontendNtB5_15FunctionBuilder8finalize
+__RNvMs9_NtNtCs6QZsINA9Usy_18cranelift_frontend8frontend10safepointsNtB5_16SafepointSpiller5clear
+__RNvMs7_NtNtCs6QZsINA9Usy_18cranelift_frontend8frontend10safepointsNtB5_16LivenessAnalysis5clear
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtCsdq5mISKDh0s_5alloc6string6StringECs7oIyYg99ywV_16cranelift_module
+__RNvXs3_NtNtCs2NHf1dqD6Ob_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCs7oIyYg99ywV_16cranelift_module
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCsdq5mISKDh0s_5alloc6string6StringNtNtCs7oIyYg99ywV_16cranelift_module6module12FuncOrDataIdEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1s_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB1w_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs7oIyYg99ywV_16cranelift_module6module19FunctionDeclarationE8grow_oneBQ_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner14grow_amortizedCs7oIyYg99ywV_16cranelift_module
+__RNvXs1_NtCs3j6f2J4rC56_13cranelift_jit7backendNtB5_9JITModuleNtNtCs7oIyYg99ywV_16cranelift_module6module6Module34define_function_with_control_plane
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtCsdq5mISKDh0s_5alloc5boxed3BoxDNtNtCs9TUm298FobZ_17cranelift_codegen6timing8ProfilerEL_EEuE16get_or_init_slowNvNvNtB2r_7enabled8PROFILER27___rust_std_internal_init_fnEB2t_
+__RNvXs1_NtNtCs9TUm298FobZ_17cranelift_codegen6timing7enabledNtB7_15DefaultProfilerNtB7_8Profiler10start_pass
+__RNvMNtCs9TUm298FobZ_17cranelift_codegen7contextNtB2_7Context8optimize
+__RNvNtCs9TUm298FobZ_17cranelift_codegen9legalizer9imm_const
+__RNvXs0_NtCs9TUm298FobZ_17cranelift_codegen6cursorQNtB5_10FuncCursorNtNtNtB7_2ir7builder16InstInserterBase17insert_built_inst
+__RNvMs4_NtNtCs9TUm298FobZ_17cranelift_codegen2ir6layoutNtB5_6Layout11insert_inst
+__RNvMs_NtCs9TUm298FobZ_17cranelift_codegen9flowgraphNtB4_16ControlFlowGraph7compute
+__RNvMNtCsihC70bBfSbG_17cranelift_bforest4poolINtB2_8NodePoolINtNtB4_3map8MapTypesNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities4InstNtB1i_5BlockEE10alloc_nodeB1m_
+__RNvMs0_NtCsihC70bBfSbG_17cranelift_bforest4nodeINtB5_8NodeDataINtNtB7_3set8SetTypesNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockEE15try_leaf_insertB1p_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxDNtNtB4_3any3AnyEL_EECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen6timing7enabled18DefaultTimingTokenEBH_
+__RNvXs2_NtNtCs9TUm298FobZ_17cranelift_codegen6timing7enabledNtB5_18DefaultTimingTokenNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtNtCs9TUm298FobZ_17cranelift_codegen6timing7enabled9PassTimesEzE16get_or_init_slowNvNvB1R_9PASS_TIME27___rust_std_internal_init_fnEB1V_
+__RNvMs2_NtCs9TUm298FobZ_17cranelift_codegen14dominator_treeNtB5_13DominatorTree7compute
+__RNvMs_NtCsihC70bBfSbG_17cranelift_bforest4pathINtB4_4PathINtNtB6_3map8MapTypesNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities4InstNtB1g_5BlockEE5firstB1k_
+__RNvMs_NtCsihC70bBfSbG_17cranelift_bforest4pathINtB4_4PathINtNtB6_3map8MapTypesNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities4InstNtB1g_5BlockEE9next_nodeB1k_
+__RNvMs0_NtCsh05dHvV41uz_16cranelift_bitset8compoundNtB5_14CompoundBitSet19try_ensure_capacityCs9TUm298FobZ_17cranelift_codegen
+__RNvNtCs9TUm298FobZ_17cranelift_codegen20remove_constant_phis23do_remove_constant_phis
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockNtNtB13_20remove_constant_phis12BlockSummaryE20resize_for_index_mutB13_
+__RNvMsA_NtNtCs9TUm298FobZ_17cranelift_codegen2ir12instructionsNtB5_15InstructionData13arguments_mut
+__RNvMs_NtCs9TUm298FobZ_17cranelift_codegen14alias_analysisNtB4_13AliasAnalysis3new
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockuNtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE6insertBT_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockNtNtBW_14alias_analysis10LastStoresEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1L_NtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE0EBW_
+__RNvMNtCs9TUm298FobZ_17cranelift_codegen14alias_analysisNtB2_10LastStores6update
+__RNCNvMs_NtCs9TUm298FobZ_17cranelift_codegen14alias_analysisNtB6_13AliasAnalysis26compute_block_input_statess_0B8_
+__RNvMs0_NtCs9TUm298FobZ_17cranelift_codegen6egraphNtB5_10EgraphPass3run
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueBX_E20resize_for_index_mutB13_
+__RNvNtCs9TUm298FobZ_17cranelift_codegen15inst_predicates18is_pure_for_egraph
+__RNvMsA_NtNtCs9TUm298FobZ_17cranelift_codegen2ir12instructionsNtB5_15InstructionData9arguments
+__RNvMs_NtNtCs9TUm298FobZ_17cranelift_codegen6egraph4costNtB4_4Cost9of_opcode
+__RNvMNtCs9TUm298FobZ_17cranelift_codegen14alias_analysisNtB2_10LastStores14get_last_store
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapNtNtCs9TUm298FobZ_17cranelift_codegen14alias_analysis9MemoryLocTNtNtNtBR_2ir8entities4InstNtB1R_5ValueENtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE6insertBR_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCs9TUm298FobZ_17cranelift_codegen14alias_analysis9MemoryLocTNtNtNtBU_2ir8entities4InstNtB1U_5ValueEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1R_NtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE0EBU_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9TUm298FobZ_17cranelift_codegen6egraph11OptimizeCtxEBF_
+__RNvMs_NtCs9TUm298FobZ_17cranelift_codegen6egraphNtB4_11OptimizeCtx17insert_pure_enode
+__RNvMsb_NtNtCs9TUm298FobZ_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph12ctrl_typevar
+__RINvXs2_NtCs9TUm298FobZ_17cranelift_codegen6egraphNtB6_10GVNContextINtNtB8_7ctxhash7CtxHashTNtNtNtB8_2ir5types4TypeNtNtB1w_12instructions15InstructionDataEE8ctx_hashNtCs4q76Ca8CNvS_10rustc_hash8FxHasherEB8_
+__RINvNtNtCs9TUm298FobZ_17cranelift_codegen4opts14generated_code20constructor_simplifyNtB4_11IsleContextINtCsbLrsMrHhqlD_8smallvec8SmallVecANtNtNtB6_2ir8entities5Valuej5_EEB6_
+__RNvXs2_NtCs9TUm298FobZ_17cranelift_codegen4optsNtB5_16InstDataEtorIterNtNtB5_14generated_code11ContextIter4next
+__RNvMs6_NtNtCs9TUm298FobZ_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph9value_def
+__RINvXss_CsbLrsMrHhqlD_8smallvecINtB6_8SmallVecANtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5Valuej5_EINtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits7collect6ExtendBJ_E6extendINtNtB1R_6option6OptionBJ_EEBP_
+__RNvMs4_NtNtCs9TUm298FobZ_17cranelift_codegen2ir6layoutNtB5_6Layout11remove_inst
+__RNvMsA_NtNtCs9TUm298FobZ_17cranelift_codegen2ir12instructionsNtB5_15InstructionData2eq
+__RNvMsA_NtNtCs9TUm298FobZ_17cranelift_codegen2ir12instructionsNtB5_15InstructionData15typevar_operand
+__RNvXs5_NtCs9TUm298FobZ_17cranelift_codegen4optsNtB5_18MaybeUnaryEtorIterNtNtB5_14generated_code11ContextIter4next
+__RINvNtNtCs9TUm298FobZ_17cranelift_codegen4opts14generated_code18constructor_truthyNtB4_11IsleContextINtB2_18ContextIterWrapperINtCsbLrsMrHhqlD_8smallvec8SmallVecANtNtNtB6_2ir8entities5Valuej8_EB1i_EEB6_
+__RNvMs0_NtNtCs9TUm298FobZ_17cranelift_codegen6egraph9elaborateNtB5_10Elaborator9elaborate
+__RNvXs_NtCs9TUm298FobZ_17cranelift_codegen10traversalsNtB4_7DfsIterNtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits8iterator8Iterator4next
+__RNvXs_NtNtNtCs2NHf1dqD6Ob_4core4iter8adapters5chainINtB4_5ChainIBO_INtNtB6_6copied6CopiedINtNtNtBa_5slice4iter4IterNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueEEINtNtB6_7flatten7FlatMapIB1q_NtNtB1T_12instructions9BlockCallEINtNtB6_10filter_map9FilterMapINtNtB6_3map3MapB1p_NCNvMB3h_B3f_4args0ENCNCNvMsb_NtB1T_3dfgNtB54_13DataFlowGraph11inst_values00ENCB4Y_0EEIB2N_INtNtBa_6option8IntoIterNtB1R_14ExceptionTableEIB3N_IB1q_NtNtB1T_15exception_table26InternalExceptionTableItemENCNvMB70_NtB70_18ExceptionTableData8contexts0ENCB4Y_s_0EENtNtNtB8_6traits8iterator8Iterator9size_hintB1V_
+__RNvMs0_NtNtCs9TUm298FobZ_17cranelift_codegen6egraph9elaborateNtB5_10Elaborator15maybe_remat_arg
+__RINvMs0_NtCs9TUm298FobZ_17cranelift_codegen15scoped_hash_mapINtB6_13ScopedHashMapNtNtNtB8_2ir8entities5ValueNtNtNtB8_6egraph9elaborate15ElaboratedValueE27insert_if_absent_with_depthNtNtB8_7ctxhash7NullCtxEB8_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTTNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockNtBT_5ValueEB1M_EE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1M_NtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE0EBX_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueNtNtNtB13_6egraph9elaborate9BestEntryE20resize_for_index_mutB13_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen6egraph9elaborate10ElaboratorEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9TUm298FobZ_17cranelift_codegen14alias_analysis13AliasAnalysisEBF_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir7extfunc9SignatureNtNtNtBW_8machinst3abi3SigEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1O_NtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE0EBW_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities6SigRefINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtNtB13_8machinst3abi3SigEE20resize_for_index_mutB13_
+__RNvMs5_NtNtCs9TUm298FobZ_17cranelift_codegen8machinst3abiNtB5_6SigSet21abi_sig_for_signature
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9TUm298FobZ_17cranelift_codegen6result12CodegenErrorEBF_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlocklE20resize_for_index_mutB13_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockINtNtNtCs2NHf1dqD6Ob_4core3ops5range5RangejEE20resize_for_index_mutB13_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst10blockorder12LoweredBlockE8grow_oneBS_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5BlockNtNtCshr3odvgaWuB_9regalloc25index5BlockE20resize_for_index_mutB13_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst10blockorder12LoweredBlockNtNtCshr3odvgaWuB_9regalloc25index5BlockEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B22_NtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE0EBW_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueINtNtNtB13_8machinst9valueregs9ValueRegsNtNtB1X_3reg3RegEE20resize_for_index_mutB13_
+__RNvNtCs9TUm298FobZ_17cranelift_codegen15inst_predicates24has_lowering_side_effect
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities4InstyEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_yNtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE0EBW_
+__RINvNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters5chain17and_then_or_clearINtB2_5ChainINtNtB4_6copied6CopiedINtNtNtB8_5slice4iter4IterNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueEEINtNtB4_7flatten7FlatMapIB1D_NtNtB26_12instructions9BlockCallEINtNtB4_10filter_map9FilterMapINtNtB4_3map3MapB1C_NCNvMB3u_B3s_4args0ENCNCNvMsb_NtB26_3dfgNtB5h_13DataFlowGraph11inst_values00ENCB5b_0EEB22_NvYB14_NtNtNtB6_6traits8iterator8Iterator4nextEB28_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueNtNtNtB13_8machinst5lower13ValueUseStateE20resize_for_index_mutB13_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecAINtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst9valueregs9ValueRegsNtNtBN_3reg3RegEj2_E8try_growBP_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCshr3odvgaWuB_9regalloc25index4InstNtBU_7PRegSetEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1t_NtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE0ECs9TUm298FobZ_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures8VRegDataE8grow_oneBS_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCshr3odvgaWuB_9regalloc2
+__RNvMs4_NtNtCsdq5mISKDh0s_5alloc11collections9vec_dequeINtB5_8VecDequeNtNtCshr3odvgaWuB_9regalloc25index5BlockE4growCs9TUm298FobZ_17cranelift_codegen
+__RNvMs6_NtNtCshr3odvgaWuB_9regalloc23ion15data_structuresNtB5_10LiveRanges3add
+__RNvXsw_Cshr3odvgaWuB_9regalloc2NtB5_4BumpNtNtNtCsvqOCrJ9GJK_14allocator_api26stable5alloc9Allocator8allocate
+__RNvMs7_CslV3gF6dMSUa_7bumpaloNtB5_4Bump17alloc_layout_slowCshr3odvgaWuB_9regalloc2
+__RNvMsi_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8BTreeMapNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures12LiveRangeKeyNtB19_14LiveRangeIndexE6insertCs9TUm298FobZ_17cranelift_codegen
+__RNvMs7_NtNtCshr3odvgaWuB_9regalloc23ion15data_structuresNtB5_11LiveBundles3add
+__RNvMs_NtNtCsvqOCrJ9GJK_14allocator_api26stable3vecINtB4_3VecNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures18LiveRangeListEntryNtB12_4BumpE6insertCs9TUm298FobZ_17cranelift_codegen
+__RNvXs0_NtNtCshr3odvgaWuB_9regalloc23ion13reg_traversalNtB5_16RegTraversalIterNtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits8iterator8Iterator4next
+__RINvMs_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree6removeINtNtB7_4node6HandleINtBY_7NodeRefNtNtBY_6marker3MutNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures12LiveRangeKeyNtB1N_14LiveRangeIndexNtB1v_4LeafENtB1v_2KVE14remove_leaf_kvNCNvMs5_NtNtB7_3map5entryINtB3U_13OccupiedEntryB1L_B2O_E9remove_kv0NtNtBb_5alloc6GlobalECs9TUm298FobZ_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures12InsertedMoveE8grow_oneBS_
+__RNvMNtCshr3odvgaWuB_9regalloc25movesINtB2_13ParallelMovesINtNtCs2NHf1dqD6Ob_4core6option6OptionNtB4_4VRegEE7resolveCs9TUm298FobZ_17cranelift_codegen
+__RNvMNtNtCshr3odvgaWuB_9regalloc23ion15redundant_movesNtB2_23RedundantMoveEliminator11clear_alloc
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCshr3odvgaWuB_9regalloc23ion15redundant_moves23RedundantMoveEliminatorECs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCshr3odvgaWuB_9regalloc23cfg7CFGInfoECs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCshr3odvgaWuB_9regalloc28indexset8IndexSetEECs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures10LiveRangesECs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures11LiveBundlesECs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures5VRegsECs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures13SpillSlotDataEECs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures14SpillSetRangesEECs9TUm298FobZ_17cranelift_codegen
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtCslV3gF6dMSUa_7bumpalo4BumpE9drop_slowCshr3odvgaWuB_9regalloc2
+__RNvMse_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecANtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst6buffer9MachLabelj10_E6resizeBO_
+__RNvMNtNtCs9TUm298FobZ_17cranelift_codegen8machinst3regNtB2_3Reg5class
+__RNvMsr_Cshr3odvgaWuB_9regalloc2NtB5_6Output11inst_allocs
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst6buffer10MachBranchEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtCsbLrsMrHhqlD_8smallvec8IntoIterANtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst6buffer13MachLabelTrapj10_EEB1h_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst10blockorder18BlockLoweringOrderEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst5vcode14VCodeConstantsEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsjh7C7x83D1_16cranelift_entity7primary10PrimaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst5vcode13VCodeConstantNtB1x_17VCodeConstantDataEEB1B_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst3abi6SigSetEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst3abi6ABIArgEEB1e_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCshr3odvgaWuB_9regalloc26OutputECs9TUm298FobZ_17cranelift_codegen
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtCs7oIyYg99ywV_16cranelift_module6module6FuncIdINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs3j6f2J4rC56_13cranelift_jit13compiled_blob12CompiledBlobEE20resize_for_index_mutB2r_
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs3j6f2J4rC56_13cranelift_jit
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs3j6f2J4rC56_13cranelift_jit
+__RNvXs3_NtNtCs3j6f2J4rC56_13cranelift_jit6memory6systemNtB5_20SystemMemoryProviderNtB7_17JITMemoryProvider8allocate
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs3j6f2J4rC56_13cranelift_jit6memory6system6PtrLenE8grow_oneBS_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once9call_onceNCNvNtCskLlwcGQiugH_6region4page4size0E0Cs3j6f2J4rC56_13cranelift_jit
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTjjNtNtCs7oIyYg99ywV_16cranelift_module6module6FuncIdEE8grow_oneCs3j6f2J4rC56_13cranelift_jit
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs7oIyYg99ywV_16cranelift_module6module6FuncIdE8grow_oneCs3j6f2J4rC56_13cranelift_jit
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir7extfunc9SignatureEBH_
+__RNvMs0_NtCs3j6f2J4rC56_13cranelift_jit7backendNtB5_9JITModule20finalize_definitions
+__RNvXs3_NtNtCs3j6f2J4rC56_13cranelift_jit6memory6systemNtB5_20SystemMemoryProviderNtB7_17JITMemoryProvider8finalize
+__RNvMs0_NtCs3j6f2J4rC56_13cranelift_jit7backendNtB5_9JITModule22get_finalized_function
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs4ZIJX5XqtQk_11weavepy_jit5value7JitTypeEENtNtBK_5clone5Clone5cloneB1l_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir11GlobalGuardENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir12IterLoopMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir13CompSavedMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir12AttrSiteMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir14MethodSiteMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir9StrMethodENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir13MathGuardMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir8OsrEntryENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit6engine14is_scalar_leaf
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir5TFuncEBF_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir6TBlockEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir11GlobalGuardEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir12AttrSiteMetaEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir12AttrSiteMetaEBF_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir13MathGuardMetaEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir8OsrEntryEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze9PathArenaECs5bNXvwCVaF0_10weavepy_vm
+__RNCNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212warm_compile0s3_0B7_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215attr_site_guard
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215attr_site_guard0B5_
+__RNvMs6_NtCs5bNXvwCVaF0_10weavepy_vm5tier2NtB5_8JitState16native_table_for
+__RNvMs6_NtCs5bNXvwCVaF0_10weavepy_vm5tier2NtB5_8JitState23method_native_table_for
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_mod5build
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11marshal_mod8register
+__RNCINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB4_12int_to_bytesEs_0B6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins13byteorder_str
+__RNvNtNtCshJldXA6Gmzv_10num_bigint7biguint7convert20to_bitwise_digits_le
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7imp_mod22imp_extension_suffixes
+__RNvCs5bNXvwCVaF0_10weavepy_vm14spec_parent_of
+__RNvMs1w_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_6Object12interned_str
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexINtNtNtNtB7_11collections4hash3set7HashSetNtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value9SharedStrEEE10initializeNCINvB2_11get_or_initNCNvMs1w_NtB26_6objectNtB3D_6Object12interned_str0E0zEB26_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4time5build
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4time1b
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4time16compute_timezone
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4time16compute_timezone0B7_
+__RNvMNtNtCsbIMwkx66vPu_6chrono6offset3utcNtB2_3Utc3now
+__RNvMNtNtCsbIMwkx66vPu_6chrono5naive4dateNtB2_9NaiveDate25from_num_days_from_ce_opt
+__RNvMs15_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB6_11BoundMethod16is_pymethod_type
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10dict_clearINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10dict_clear
+__RNvXse_NtNtCsdq5mISKDh0s_5alloc3vec9into_iterINtB5_8IntoIterTNtNtCs5bNXvwCVaF0_10weavepy_vm6object7DictKeyNtBZ_6ObjectEENtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4dropB11_
+__RNvCs5bNXvwCVaF0_10weavepy_vm13bind_int_args
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11do_int_call
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins12b_int_compat
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins16parse_int_string
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins27transform_decimal_and_space
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4time17time_monotonic_ns
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtB7_4time7InstantE10initializeNCINvB2_11get_or_initNvMBV_BT_3nowE0zECs5bNXvwCVaF0_10weavepy_vm
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtBc_4time7InstantE10initializeNCINvB1a_11get_or_initNvMB1I_B1G_3nowE0zE0E0Cs5bNXvwCVaF0_10weavepy_vm
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21percent_mod_left_slot
+__RNvCs5bNXvwCVaF0_10weavepy_vm24percent_args_need_bridge
+__RNvCs5bNXvwCVaF0_10weavepy_vm19percent_format_with
+__RINvMs3_NtCsdq5mISKDh0s_5alloc3stre7replaceAcj2_ECs5bNXvwCVaF0_10weavepy_vm
+__RNvCs5bNXvwCVaF0_10weavepy_vm10format_int
+__RNvCs5bNXvwCVaF0_10weavepy_vm16group_int_digits
+__RNvCs5bNXvwCVaF0_10weavepy_vm19int_precision_apply
+__RNvCs5bNXvwCVaF0_10weavepy_vm9with_sign
+__RNvCs5bNXvwCVaF0_10weavepy_vm15apply_alignment
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter8do_print
+__RNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB7_11Interpreter8do_prints_0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4math5build
+__RNvCs5bNXvwCVaF0_10weavepy_vm20checked_repeat_count
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4math7builtin
+__RINvCs5bNXvwCVaF0_10weavepy_vm13try_alloc_vecNtNtB2_6object6ObjectEB2_
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze17const_slice_shape
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze15bin_result_type
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins7b_bytesINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib14operator_accel5build
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10b_bytes_kw
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13itertools_mod5build
+__RNvYNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13itertools_mod31register_classmethod_descriptorINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib14operator_accel5build0B7_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins21bytes_from_source_obj
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib14datetime_accel5build
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13itertools_mod31register_classmethod_descriptor
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir6TBlockEEEB1y_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6result6ResultNtNtCs5bNXvwCVaF0_10weavepy_vm6object10PyIteratorNtNtB11_5error12RuntimeErrorEEB11_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins29bytes_from_iterable_reentrant
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib14datetime_accel15timedelta_parts
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15byte_item_value
+__RNvCs5bNXvwCVaF0_10weavepy_vm9float_pow
+__RINvCs5bNXvwCVaF0_10weavepy_vm13try_alloc_vechEB2_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir14MethodSiteMetaEEEB1y_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm14descr_registry10descr_type
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm5types12InstancePlanE9drop_slowBK_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11b_enumerateINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11b_enumerate
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins22coexist_builtin_dunder
+__RNvCs5bNXvwCVaF0_10weavepy_vm16class_dunder_new
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell4CelljEE4withNCNvCs5bNXvwCVaF0_10weavepy_vm13is_object_new0jEB1E_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir14ResolvedGlobalEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvCs5bNXvwCVaF0_10weavepy_vm17is_union_eligible
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter22deopt_store_attr_taken
+__RNvCs5bNXvwCVaF0_10weavepy_vm18is_union_initiator
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze4PlanEBF_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib14datetime_accel11date_fields
+__RNvCs5bNXvwCVaF0_10weavepy_vm17make_pep604_union
+__RNCNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212warm_compile0s6_0B7_
+__RNCNvCs5bNXvwCVaF0_10weavepy_vm17make_pep604_union0B3_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter25push_pending_caller_shell
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier216probe_param_lane
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib14datetime_accel11time_fields
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze7pop_val
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native7install
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter12do_hash_call
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze9lower_bin
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier219resolve_plain_dicts
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11hash_object
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215classify_global
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRThReEECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier216probe_class_ctor
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTThReENtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_BV_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBZ_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier222probe_class_ctor_shape
+__RNvMs2_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_nativeNtB5_5Names3new
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjTmmENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native5build
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native5build0B7_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjTmmmENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4ZIJX5XqtQk_11weavepy_jit
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer14div_guard_cond
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13leaf_builtins4push
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4ZIJX5XqtQk_11weavepy_jit
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer18floor_needs_adjust
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjmEE14reserve_rehashNCINvNtB8_3map11make_hasherjmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4ZIJX5XqtQk_11weavepy_jit
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze7PairSrcNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBS_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11do_sum_call
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_upperINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins22sum_small_int_sequence
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze7PairSrcEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_upper
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir12ListLoopMetaE8grow_oneBQ_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins20sum_numeric_sequence
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins19sum_exact_int_range
+__RNvMNtCseIGkHRfguhS_9hashbrown11rustc_entryINtNtB4_3map7HashMapmINtNtCsdq5mISKDh0s_5alloc3vec3VecjENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE11rustc_entryCs4ZIJX5XqtQk_11weavepy_jit
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_titleINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_title
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCsdq5mISKDh0s_5alloc3vec3VecThReNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEB1y_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCsdq5mISKDh0s_5alloc3vec3VecThReNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B2l_
+__RNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB7_11Interpreter21percent_mod_left_slot0B7_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir11GlobalGuardE8grow_oneBQ_
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtNtNtBa_11collections4hash3map7HashMapReNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEE4withNCNvNtB1G_8builtins12vm_intrinsic0INtNtCs2NHf1dqD6Ob_4core6option6OptionB1C_EEB1G_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmTmNtNtCs4ZIJX5XqtQk_11weavepy_jit5value7JitTypeENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBU_
+__RNvXs7_NtCs9TUm298FobZ_17cranelift_codegen4optsNtB5_11IsleContextNtNtB5_14generated_code7Context19iconst_sextend_etor
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtNtNtBe_11collections4hash3map7HashMapReNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEuE16get_or_init_slowNvNvNvNtB20_8builtins12vm_intrinsic5TABLE27___rust_std_internal_init_fnEB20_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmTmNtNtCs4ZIJX5XqtQk_11weavepy_jit5value7JitTypeEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBX_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueNtBZ_5BlockE20resize_for_index_mutB13_
+__RNCNvNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins12vm_intrinsic5build0B7_
+__RNvMNtCsjh7C7x83D1_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValuehE20resize_for_index_mutB13_
+__RINvNtNtCs9TUm298FobZ_17cranelift_codegen4opts14generated_code25constructor_ty_shift_maskNtB4_11IsleContextEB6_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_countINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9str_count
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecANtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5Valuej5_E8try_growBO_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins13b_set_tp_name
+__RINvNtNtCs9TUm298FobZ_17cranelift_codegen4opts14generated_code20constructor_iconst_uNtB4_11IsleContextEB6_
+__RINvNtNtCs9TUm298FobZ_17cranelift_codegen4opts14generated_code20constructor_iconst_sNtB4_11IsleContextEB6_
+__RINvXss_CsbLrsMrHhqlD_8smallvecINtB6_8SmallVecANtNtNtCs9TUm298FobZ_17cranelift_codegen4opts14generated_code26SkeletonInstSimplificationj5_EINtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits7collect6ExtendBJ_E6extendINtNtB2m_6option6OptionBJ_EEBP_
+__RINvNtNtCs9TUm298FobZ_17cranelift_codegen4opts14generated_code24constructor_compose_icmpNtB4_11IsleContextEB6_
+__RNvMs2_NtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB5_9CtorState10note_store
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4math13make_checked10B7_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst3abi6ABIArgE8grow_oneBS_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4math6to_f64
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14coerce_f64_opt
+__RNvMNtCs6KxzipSu1yP_3std3f64d3sin
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215probe_list_lane
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215probe_dict_lane
+__RNvNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4math14freshen_result7freshen
+__RNvMNtCs6KxzipSu1yP_3std3f64d3cos
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4math7builtin0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib4math9math_sqrt
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native13plain_drop_ok
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze14boundary_stack
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable7ipnsortjNvYjNtNtB8_3cmp10PartialOrd2ltECs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze9cmp_check
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable9quicksort9quicksortjNvYjNtNtBa_3cmp10PartialOrd2ltECs4ZIJX5XqtQk_11weavepy_jit
+__RNvXNtCs2NHf1dqD6Ob_4core3anyNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native5StateNtB2_3Any7type_idBx_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native11verify_slow
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze12infer_append
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native6dt_add
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter12do_bool_call
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native9dt_fields
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmTNtNtCs4ZIJX5XqtQk_11weavepy_jit5value7JitTypeBP_ENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBT_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native9td_fields
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter20native_value_payload
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13class_ns_load
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze10seed_entry
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native10valid_date
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14class_ns_store
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native8dt_shift
+__RNvMs_NtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB4_4Plan9hidden_at
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter9stringify
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native7ymd2ord
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native7ord2ymd
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze22resolve_dict_container
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native6new_dt
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native14instance_fixed
+__RNvMsg_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_11SlotStorage11from_layout
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir14CalleeSpanMetaE8grow_oneBQ_
+__RNvMs3_NtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB5_2SE8is_plain
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze22entry_unassigned_reads
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native6dt_sub
+__RNvMs2_NtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB5_9CtorState10field_lane
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtCsdq5mISKDh0s_5alloc5boxed3BoxDNtNtCs9TUm298FobZ_17cranelift_codegen6timing8ProfilerEL_EEEB2j_
+__RNvMs2_NtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB5_9CtorState12stored_field
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native6new_td
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface11unwrap_shim0B5_
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface11unwrap_shim14check_receiver
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native6td_mul
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native6td_cmp
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze17dyn_attr_fallback
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins12dict_setitemINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxSNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEB1f_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins12dict_setitem
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native12date_weekday
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native14date_toordinal
+__RNvYNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types22install_object_dunders16object_class_getINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native11date_fields
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types22install_object_dunders16object_class_get
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12weakref_real14proxy_referent
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier214probe_obj_live
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter25materialize_class_mapping
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native12dt_isoformat
+__RINvYNtNtCs6QZsINA9Usy_18cranelift_frontend8frontend15FuncInstBuilderNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir7builder11InstBuilder4jumpRINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB19_12instructions8BlockArgEECs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native5tz_of
+__RNvXsZ_NtCsdq5mISKDh0s_5alloc6stringNtB5_6StringNtNtCs2NHf1dqD6Ob_4core3fmt5Write10write_char
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native13format_offset
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native16dt_fromisoformat
+__RNvCs5bNXvwCVaF0_10weavepy_vm14intersect_sets
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native9parse_hms
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21try_dict_from_mapping
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins6b_dictINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze14CompHeaderPlanEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins6b_dict
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native11dt_strftime
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native16strftime_numeric
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib15datetime_native8date_sub
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer5truth
+__RINvYNtNtCs6QZsINA9Usy_18cranelift_frontend8frontend15FuncInstBuilderNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir7builder11InstBuilder4brifRINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB19_12instructions8BlockArgEB2b_ECs4ZIJX5XqtQk_11weavepy_jit
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecANtNtCshr3odvgaWuB_9regalloc25index5Blockj10_E21reserve_one_uncheckedCs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm5tier211MethodEntryEEB1c_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecANtNtCshr3odvgaWuB_9regalloc25index5Blockj10_E8try_growCs9TUm298FobZ_17cranelift_codegen
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer14pin_helper_sig
+__RNvYNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types20install_type_dunders12type_delattrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecANtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst6buffer10MachBranchj4_E21reserve_one_uncheckedBO_
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types20install_type_dunders12type_delattr
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer14stage_elements
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer15quad_helper_sig
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjThjENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4ZIJX5XqtQk_11weavepy_jit
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjThjEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4ZIJX5XqtQk_11weavepy_jit
+__RNCNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212warm_compile00B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native12deque_append
+__RNvMsi_NtCs5bNXvwCVaF0_10weavepy_vm5typesNtB5_11SlotStorage14get_hinted_mut
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native13deque_popleft
+__RINvYNtNtCs6QZsINA9Usy_18cranelift_frontend8frontend15FuncInstBuilderNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir7builder11InstBuilder4brifRINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB19_12instructions8BlockArgERAB2J_j0_ECs4ZIJX5XqtQk_11weavepy_jit
+__RNvYNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir7builder14ReplaceBuilderNtB4_11InstBuilder4iaddB8_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native7head_of
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs8FvlOjbra0L_14target_lexicon6triple6TripleECs9TUm298FobZ_17cranelift_codegen
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10b_callableINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtBG_5boxed3BoxDG_INtNtNtB4_3ops8function2FnTRL0_eEEp6OutputINtNtB4_6option6OptionPhENtNtB4_6marker4SendEL_EEECs3j6f2J4rC56_13cranelift_jit
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native13deque_getitem
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18object_is_callable
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins4b_idINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxDG_INtNtNtB4_3ops8function2FnTRL0_eEEp6OutputINtNtB4_6option6OptionPhENtNtB4_6marker4SendEL_EECs3j6f2J4rC56_13cranelift_jit
+__RNvMs0_NtCsihC70bBfSbG_17cranelift_bforest4nodeINtB5_8NodeDataINtNtB7_3map8MapTypesNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities4InstNtB1l_5BlockEE15try_leaf_insertB1p_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins19coerce_index_object
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native8receiver
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native10DequeStateEBH_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins15object_identity
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native10deque_bool
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs3j6f2J4rC56_13cranelift_jit6memory6system20SystemMemoryProviderEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs3j6f2J4rC56_13cranelift_jit6memory6system6MemoryEBH_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities5ValueNtNtBW_20remove_constant_phis13AbstractValueEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1L_NtCs4q76Ca8CNvS_10rustc_hash13FxBuildHasherE0EBW_
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze22resolve_list_container
+__RNvMs1_NtCscifjXFcM73a_16weavepy_compiler8bytecodeNtB5_6OpCode4name
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs3j6f2J4rC56_13cranelift_jit6memory6system6PtrLenEEB1e_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs7oIyYg99ywV_16cranelift_module6module18ModuleDeclarationsECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native20deque_iterator_index
+__RNvMNtCseIGkHRfguhS_9hashbrown11rustc_entryINtNtB4_3map7HashMapmmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE11rustc_entryCs4ZIJX5XqtQk_11weavepy_jit
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins16coerce_index_i64
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins20try_coerce_index_i64
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native19deque_iterator_next
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native10deque_next
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm6object14SlotDescriptorE9drop_slowBK_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter9op_binary
+__RNvMs_NtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB4_4Plan18comp_lane_for_body
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native16deque_appendleft
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCs5bNXvwCVaF0_10weavepy_vm5types10TypeObjectE9drop_slowBK_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCsdq5mISKDh0s_5alloc3vec6splice6SpliceINtNtNtNtB4_4iter7sources8repeat_n7RepeatNNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEEB24_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11do_len_call
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsjh7C7x83D1_16cranelift_entity3map12SecondaryMapNtNtCs7oIyYg99ywV_16cranelift_module6module6DataIdINtNtB4_6option6OptionNtNtCs3j6f2J4rC56_13cranelift_jit13compiled_blob12CompiledBlobEEEB2H_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib18collections_native9deque_len
+__RNCNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212warm_compile0s0_0B7_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter16core_pure_method
+__RNvCs5bNXvwCVaF0_10weavepy_vm17coerce_len_result
+__RNCNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212warm_compile0s1_0B7_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs9TUm298FobZ_17cranelift_codegen13loop_analysis8LoopDataE8grow_oneBQ_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9TUm298FobZ_17cranelift_codegen7context7ContextECs5bNXvwCVaF0_10weavepy_vm
+__RNCNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212warm_compile0s7_0B7_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir13CompSavedMetaNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBS_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir13CompSavedMetaEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec21spec_from_iter_nestedINtB6_3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze2SEEINtB4_18SpecFromIterNestedB13_INtNtB6_5drain5DrainB13_EE9from_iterB17_
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze14escape_verdict
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8constant12ConstantPoolECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsjh7C7x83D1_16cranelift_entity7primary10PrimaryMapNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8entities14ExceptionTableNtNtB1z_15exception_table18ExceptionTableDataEEB1B_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir6layout6LayoutEBH_
+__RINvXs_NtCs9TUm298FobZ_17cranelift_codegen7ctxhashNtB5_7NullCtxINtB5_7CtxHashTNtNtNtB7_2ir5types4TypeNtNtB1i_12instructions15InstructionDataEE8ctx_hashNtCs4q76Ca8CNvS_10rustc_hash8FxHasherEB7_
+__RNvXsc_NtNtCs2NHf1dqD6Ob_4core5slice5indexTINtNtNtB9_3ops5range5BoundjEBF_EINtB5_10SliceIndexSNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures18LiveRangeListEntryE5indexCs9TUm298FobZ_17cranelift_codegen
+__RNvYNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir7builder14ReplaceBuilderNtB4_11InstBuilder4bxorB8_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir10debug_tags9DebugTagsEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs9TUm298FobZ_17cranelift_codegen2ir8function18FunctionParametersEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9TUm298FobZ_17cranelift_codegen9flowgraph16ControlFlowGraphEBF_
+__RNCNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212warm_compile0sa_0B7_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier211guards_hold
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier28take_u64
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9TUm298FobZ_17cranelift_codegen14dominator_tree13DominatorTreeECs5bNXvwCVaF0_10weavepy_vm
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCs5bNXvwCVaF0_10weavepy_vm5tier27JitBufsEuE16get_or_init_slowNvNvB1R_8JIT_BUFS27___rust_std_internal_init_fnEB1T_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier28take_u32
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCs5bNXvwCVaF0_10weavepy_vm5tier28JitStateEE4withNCNvB1x_19current_compile_gen0yEB1z_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier214wpjit_attr_get
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier211unpack_pins
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCs9TUm298FobZ_17cranelift_codegen8machinst16CompiledCodeBaseNtNtB10_6buffer5FinalEEECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs6QZsINA9Usy_18cranelift_frontend8frontend22FunctionBuilderContextECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecmEE8grow_oneCscifjXFcM73a_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5tier27CallCtxEBF_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm5tier23PinEEB1c_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecATmNtNtNtCs9TUm298FobZ_17cranelift_codegen3isa6unwind10UnwindInstEj8_E21reserve_one_uncheckedBQ_
+__RNvYNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13itertools_mod10count_coreINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13itertools_mod10count_core
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real13allocate_lock
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real16make_lock_object
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13functools_mod5build
+__RNCNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13functools_mod8lru_type00B9_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real16new_rlock_object
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtNtNtBa_11collections4hash3map7HashMapReNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEEB1Q_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter11do_str_call
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter12load_special
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys10sys_intern
+__RNCNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB7_11Interpreter12load_special0B7_
+__RINvMs10_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree4nodeINtB7_16BalancingContextNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures12LiveRangeKeyNtB1l_14LiveRangeIndexE25merge_tracking_child_edgeNtNtBd_5alloc6GlobalECs9TUm298FobZ_17cranelift_codegen
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real26instance_dunder_trampoline0B7_
+__RNvMs10_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree4nodeINtB6_16BalancingContextNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures12LiveRangeKeyNtB1k_14LiveRangeIndexE15bulk_steal_leftCs9TUm298FobZ_17cranelift_codegen
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real16make_lock_objects1_0B7_
+__RNvMsA_NtCs5bNXvwCVaF0_10weavepy_vm4syncNtB5_8RealLock11try_acquire
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins16str_isidentifierINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier214entry_local_ok
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier220decompose_live_loops
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins16str_isidentifier
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real16make_lock_objects2_0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real17thread_handle_new
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins12obj_containsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real25make_thread_handle_object
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCs6dvEX9a1rCm_8lock_api5mutex5MutexNtNtCswHAcCYaqJh_11parking_lot9raw_mutex8RawMutexINtNtNtNtB7_11collections4hash3map7HashMapyINtNtCsdq5mISKDh0s_5alloc4sync3ArcNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real17ThreadHandleStateEEEE10initializeNCINvB2_11get_or_initNCNvB3B_15handle_registry0E0zEB3F_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier214enter_compiled
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTyINtNtCsdq5mISKDh0s_5alloc4sync3ArcNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real17ThreadHandleStateEEE14reserve_rehashNCINvNtB8_3map11make_hasheryBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB1v_
+__RNvMs10_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree4nodeINtB6_16BalancingContextNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures12LiveRangeKeyNtB1k_14LiveRangeIndexE16bulk_steal_rightCs9TUm298FobZ_17cranelift_codegen
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins24bytearray_receiver_guard
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm6object20ByteArrayExportGuardEEB11_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real15new_lock_object
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object19current_interp_hash
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17merge_exec_kwargs
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter12do_eval_call
+__RNvCs5bNXvwCVaF0_10weavepy_vm26check_compile_source_nulls
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtNtCs5bNXvwCVaF0_10weavepy_vm5tier29ArtifactsE9drop_slowBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5tier29ArtifactsEBF_
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtNtCs4ZIJX5XqtQk_11weavepy_jit6engine13CompiledFrameE9drop_slowCs5bNXvwCVaF0_10weavepy_vm
+__RNCINvMs18_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB9_9BuiltinFn11with_kwargsNCNvNtNtBb_6stdlib11thread_real16make_lock_object0Es_0Bb_
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser17parse_eval_module
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real16make_lock_object0B7_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4ZIJX5XqtQk_11weavepy_jit6engine13CompiledFrameECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real20resolve_acquire_args
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser18parse_eval_element
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier217wpjit_list_append
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier214wpjit_list_len
+__RNvMNtCsllk7SLrseIa_14weavepy_parser6parserNtB2_6Parser12parse_lambda
+__RNvCsllk7SLrseIa_14weavepy_parser21eval_backslash_at_eof
+__RNvNvCs5bNXvwCVaF0_10weavepy_vm32collect_compiler_syntax_warnings10visit_body
+__RNvNvCs5bNXvwCVaF0_10weavepy_vm32collect_compiler_syntax_warnings10visit_expr
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real21get_main_thread_ident
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real18make_thread_handle
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable14driftsort_mainTNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures11PosWithPrioNtB16_4EditENCINvMNtCsdq5mISKDh0s_5alloc5sliceSBZ_11sort_by_keyyNCNvMsp_B12_NtB12_5Edits4sort0E0INtNtB2m_3vec3VecBZ_EECs9TUm298FobZ_17cranelift_codegen
+__RNvNvCs5bNXvwCVaF0_10weavepy_vm32collect_compiler_syntax_warnings10visit_args
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable5drift4sortTNtNtNtCshr3odvgaWuB_9regalloc23ion15data_structures11PosWithPrioNtB13_4EditENCINvMNtCsdq5mISKDh0s_5alloc5sliceSBW_11sort_by_keyyNCNvMsp_BZ_NtBZ_5Edits4sort0E0ECs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs86VDVqQvmSJ_13weavepy_lexer5token13EscapeWarningEECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real25make_thread_handle_object0EBJ_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter25exec_eval_compile_options
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real13get_native_id
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real17make_rlock_objects6_0B7_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecANtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst10blockorder12LoweredBlockj40_E8try_growBO_
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real17make_rlock_objects7_0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10os_process19register_at_fork_kw
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir12AttrSiteMetaEECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTmNtNtCs5bNXvwCVaF0_10weavepy_vm5tier29SynthSeedEEEB1e_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5tier213CompiledEntryEBF_
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze16infer_store_attr
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB14_6string6StringENtNtCs4ZIJX5XqtQk_11weavepy_jit5value7JitTypeIBC_TB1w_mEEbEEEB1X_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecAhj400_E8try_growCs9TUm298FobZ_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser3ast8StmtKindECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser3ast9ArgumentsECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtNtCs5bNXvwCVaF0_10weavepy_vm5tier213GuardSnapshotE9drop_slowBH_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter14lazy_iter_next
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast3ArgEECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsllk7SLrseIa_14weavepy_parser3ast3ArgECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtB7_3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEE9drop_slowBX_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCsllk7SLrseIa_14weavepy_parser3ast4ExprEEECs5bNXvwCVaF0_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBW_6string6StringEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4ZIJX5XqtQk_11weavepy_jit
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtB7_3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm5tier29AttrGuardEE9drop_slowBX_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier214wpjit_attr_set
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm5tier29AttrGuardEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast4ExprEECs5bNXvwCVaF0_10weavepy_vm
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRTmNtNtCsdq5mISKDh0s_5alloc6string6StringEECs4ZIJX5XqtQk_11weavepy_jit
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtB7_3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm5tier211MethodEntryEE9drop_slowBX_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTTmNtNtCsdq5mISKDh0s_5alloc6string6StringENtNtCs4ZIJX5XqtQk_11weavepy_jit5value7JitTypeEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1v_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB1z_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCsllk7SLrseIa_14weavepy_parser3ast4ExprEECs5bNXvwCVaF0_10weavepy_vm
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecANtCshr3odvgaWuB_9regalloc210Allocationj4_E21reserve_one_uncheckedBK_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins16mappingproxy_getINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsllk7SLrseIa_14weavepy_parser3ast7KeywordEECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins16mappingproxy_get
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins21mappingproxy_delegate
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins17mappingproxy_args
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtB7_3vec3VecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm5tier212NativeCalleeEEE9drop_slowB1z_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real9get_ident
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm5tier212NativeCalleeEEEB1y_
+__RNvYNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types20install_type_dunders12type_setattrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter30winner_aware_dynamic_type_call
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types20install_type_dunders12type_setattr
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins7set_popINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNCINvMs18_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB9_9BuiltinFn11with_kwargsNvNtNtBb_6stdlib11thread_real21start_joinable_threadEs_0Bb_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins7set_pop
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys16build_with_states_0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real19spawn_python_worker
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib3sys22sys_getframemodulename
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm15thread_registry14ThreadRegistryE10initializeNCINvB2_11get_or_initNvMs0_BV_BT_3newE0zEBX_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCs5bNXvwCVaF0_10weavepy_vm15thread_registry14ThreadRegistryE10initializeNCINvB1a_11get_or_initNvMs0_B1I_B1G_3newE0zE0E0B1K_
+__RNvYNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13functools_mod12partial_callINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1R_EEE4callB8_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib13functools_mod12partial_call
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEE8grow_oneB1c_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real26lock_acquire_interruptible
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10struct_mod5build
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types20install_type_dunders12type_doc_get
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable7ipnsortNtNvMNtNtCshr3odvgaWuB_9regalloc23ion5movesINtNtB10_15data_structures3EnvpE34apply_allocations_and_insert_moves14InterBlockDestNCINvMB6_SBT_20sort_unstable_by_keyyNvMs_BV_BT_3keyE0ECs9TUm298FobZ_17cranelift_codegen
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable9quicksort9quicksortNtNvMNtNtCshr3odvgaWuB_9regalloc23ion5movesINtNtB1e_15data_structures3EnvpE34apply_allocations_and_insert_moves14InterBlockDestNCINvMB8_SB17_20sort_unstable_by_keyyNvMs_B19_B17_3keyE0ECs9TUm298FobZ_17cranelift_codegen
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real16make_lock_objects_0B7_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface20obj_contains_builtin
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10codecs_mod5build
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs5bNXvwCVaF0_10weavepy_vm3gil8GilGuardEEB1c_
+__RNvCs5bNXvwCVaF0_10weavepy_vm25detect_yield_from_subiter
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm3gil8GilGuardEBF_
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface22install_object_compare6obj_eq
+__RNvMsA_NtCs5bNXvwCVaF0_10weavepy_vm4syncNtB5_8RealLock15acquire_timeout
+__RNvMs_NtCswHAcCYaqJh_11parking_lot7condvarNtB4_7Condvar19wait_until_internal
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageNtNtCsdCPRsQxKMa_16parking_lot_core11parking_lot10ThreadDatauE16get_or_init_slowNvNvNvB1g_16with_thread_data11THREAD_DATA27___rust_std_internal_init_fnECswHAcCYaqJh_11parking_lot
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215callee_ret_lane
+__RNvMs1_NtCsdCPRsQxKMa_16parking_lot_core11parking_lotNtB5_10ThreadData3new
+__RNvMNtCsdCPRsQxKMa_16parking_lot_core11parking_lotNtB2_9HashTable3new
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyNtNtCs5bNXvwCVaF0_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_13key_cmp_scopeINtNtCs2NHf1dqD6Ob_4core6option6OptionNtBW_6ObjectENCNvMs3_BY_NtBY_11Interpreter11instantiates_0E0INtNtB2n_6result6ResultB2i_NtNtBY_5error12RuntimeErrorEEBY_
+__RNvMs_NtCs5bNXvwCVaF0_10weavepy_vm6importNtB4_11ModuleCache6remove
+__RNSNvYNCINvNtNtCs6KxzipSu1yP_3std6thread9lifecycle15spawn_uncheckedNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real19spawn_python_workers1_0uEs_0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceuE9call_once6vtableB1b_
+__RNvMs3_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_7PyFrame15locals_snapshot
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215callee_ret_infos0_0B5_
+__RINvNtNtCs6KxzipSu1yP_3std3sys9backtrace28___rust_begin_short_backtraceNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real19spawn_python_workers1_0uEB1f_
+__RNvMs3_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_7PyFrame22has_live_hidden_locals
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4ZIJX5XqtQk_11weavepy_jit7analyze9PathArenaEBF_
+__RNvMs3_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_7PyFrame14compute_locals
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable14driftsort_mainNtNtCsdq5mISKDh0s_5alloc6string6StringNvYBZ_NtNtB8_3cmp10PartialOrd2ltINtNtB13_3vec3VecBZ_EECscifjXFcM73a_16weavepy_compiler
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable5drift4sortNtNtCsdq5mISKDh0s_5alloc6string6StringNvYBW_NtNtBa_3cmp10PartialOrd2ltECscifjXFcM73a_16weavepy_compiler
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTyyEE14reserve_rehashNCINvNtB8_3map11make_hasheryyNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs5bNXvwCVaF0_10weavepy_vm
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable9quicksort9quicksortNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB15_NtNtBa_3cmp10PartialOrd2ltECscifjXFcM73a_16weavepy_compiler
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm13vm_singletons26install_seed_builtin_types
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared5pivot11median3_recNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB14_NtNtBa_3cmp10PartialOrd2ltECscifjXFcM73a_16weavepy_compiler
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort31small_sort_general_with_scratchNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB1s_NtNtBa_3cmp10PartialOrd2ltECscifjXFcM73a_16weavepy_compiler
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort12sort4_stableNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB19_NtNtBa_3cmp10PartialOrd2ltECscifjXFcM73a_16weavepy_compiler
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11str_isupperINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNCNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze15analyze_for_rets0_0B5_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11str_isupper
+__RNCNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze15analyze_for_ret0B5_
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215callee_ret_infos1_0B5_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBI_6string6StringEB1a_EECs5bNXvwCVaF0_10weavepy_vm
+__RNCNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212warm_compile0s4_0B7_
+__RNvMsf_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellNtNtB7_16weakref_registry13RegistryInnerE17try_borrow_lockedB7_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier218probe_method_entry
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215method_ret_info
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215method_ret_infos_0B5_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel5build
+__RNvYNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel10make_loadsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel10make_loads
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB1M_6string6StringEB2e_EECs5bNXvwCVaF0_10weavepy_vm
+__RNvMNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB2_13FunctionGuard3new
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBX_6string6StringEB1p_EmEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_mNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs5bNXvwCVaF0_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs5bNXvwCVaF0_10weavepy_vm5tier211MethodEntryE8grow_oneBQ_
+__RNvMs_NtCswHAcCYaqJh_11parking_lot7condvarNtB4_7Condvar15notify_one_slow
+__RNvNtCsdCPRsQxKMa_16parking_lot_core11parking_lot16lock_bucket_pair
+__RNvMs1_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB5_11ImportGuard3new
+__RNvMs1_NtCswHAcCYaqJh_11parking_lot9raw_mutexNtB5_8RawMutex9lock_slow
+__RNvMs0_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB5_12BuiltinGuard3new
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13delete_subscr
+__RNvNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel7classes8pure_get
+__RNvMs_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB4_20ModuleFunctionsGuard3new
+__RNvMs6_NtCs5bNXvwCVaF0_10weavepy_vm5tier2NtB5_8JitState21resolve_native_callee
+__RNvMs6_NtCs5bNXvwCVaF0_10weavepy_vm5tier2NtB5_8JitState19resolve_native_func
+__RNvXNtNtCs2NHf1dqD6Ob_4core4iter8adaptersINtB2_12GenericShuntINtNtB2_3map3MapINtNtNtB6_5slice4iter4IterReENCNvMs_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB1N_20ModuleFunctionsGuard3new0EINtNtB6_6option6OptionNtNtB6_7convert10InfallibleEENtNtNtB4_6traits8iterator8Iterator4nextB1R_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15pull_for_unpack
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215method_ret_infos2_0B5_
+__RNvMs_NtCs4ZIJX5XqtQk_11weavepy_jit2irNtB4_9StrMethod9from_name
+__RNvYNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel10make_dumpsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel10make_dumps
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215method_ret_infos6_0B5_
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215method_ret_infos0_0B5_
+__RNvMsf_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellAjj3_E21try_borrow_mut_lockedB7_
+__RNvMsf_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellAjj3_E17try_borrow_lockedB7_
+__RNvMs5_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB5_19ClassFunctionsGuard3new
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier220wpjit_iter_next_pair
+__RNvYNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types22install_object_dunders14object_delattrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215wpjit_bytes_get
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir14MethodSiteMetaEE6resizeB1l_
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types22install_object_dunders14object_delattr
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier216wpjit_global_obj
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24generic_delattr_instance
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit5lowerNtB2_7Lowerer22call_method_helper_sig
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier214wpjit_call_dyn
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter24call_object_with_globals
+__RNvMNtCs4ZIJX5XqtQk_11weavepy_jit7analyzeNtB2_9PathArena3seg
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier27pin_any
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared5pivot11median3_recNtNvMNtNtCshr3odvgaWuB_9regalloc23ion5movesINtNtB1b_15data_structures3EnvpE34apply_allocations_and_insert_moves14InterBlockDestNCINvMB8_SB14_20sort_unstable_by_keyyNvMs_B16_B14_3keyE0ECs9TUm298FobZ_17cranelift_codegen
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215attr_chain_step
+__RNvMsf_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellNtNtB7_16weakref_registry13RegistryInnerE21try_borrow_mut_lockedB7_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier214wpjit_list_set
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface15obj_len_builtin
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface19install_type_checks18type_subclasscheck
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCs5bNXvwCVaF0_10weavepy_vm11InterpreterEBD_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm6import11ModuleCacheEBF_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18mappingproxy_itemsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18mappingproxy_items
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueTmNtNtCs5bNXvwCVaF0_10weavepy_vm5tier29SynthSeedEEBH_
+__RNvCs5bNXvwCVaF0_10weavepy_vm16fallback_globals
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtCs5bNXvwCVaF0_10weavepy_vm9InlineActEEB1c_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7sre_mod5build
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtCs5bNXvwCVaF0_10weavepy_vm12shared_value7ThinArcNtNtB1d_13tuple_storage12TupleStorageEEEB1d_
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types14make_owned_news_0B5_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object18native_scalar_repr
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm13builtin_types14make_owned_new11reject_bool
+__RNvXsp_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecIBw_NtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEENtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4dropBN_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier216wpjit_build_list
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier217wpjit_list_repeat
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier213wpjit_call_py
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5tier211MethodEntryEBF_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215try_native_call
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier217enter_scalar_leaf
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier214wpjit_list_get
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecAmj40_E21reserve_one_uncheckedCs9TUm298FobZ_17cranelift_codegen
+__RNvCs5bNXvwCVaF0_10weavepy_vm13slice_indices
+__RINvMs_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectE5drainINtNtNtCs2NHf1dqD6Ob_4core3ops5range14RangeInclusivejEEBJ_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCs6dvEX9a1rCm_8lock_api5mutex5MutexNtNtCswHAcCYaqJh_11parking_lot9raw_mutex8RawMutexINtNtCs2NHf1dqD6Ob_4core6option6OptionIBx_NtNtCs5bNXvwCVaF0_10weavepy_vm15thread_registry11ThreadEntryEEEE9drop_slowB2R_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_ordINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecANtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst10blockorder12LoweredBlockj80_E21reserve_one_uncheckedBO_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_ord
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCs5bNXvwCVaF0_10weavepy_vm5tier27JitBufsEEB1J_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecANtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst6buffer12MachCallSitej10_E21reserve_one_uncheckedBO_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecAINtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst6buffer10MachSrcLocNtBL_7StencilEj40_E21reserve_one_uncheckedBP_
+__RNvMsc_CsbLrsMrHhqlD_8smallvecINtB5_8SmallVecAINtNtNtCs9TUm298FobZ_17cranelift_codegen8machinst6buffer10MachSrcLocNtBL_5FinalEj40_E8try_growBP_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCs5bNXvwCVaF0_10weavepy_vm4sync7GilCellINtNtB7_3vec3VecNtNtBL_5error11PyExceptionEEE9drop_slowBL_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCINvMs18_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtBL_9BuiltinFn11with_kwargsNCNvNtNtBN_6stdlib11thread_real16make_lock_object0E0EBN_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object13int_from_i128
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real25make_thread_handle_objects_0B7_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17do_list_sort_call
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4ZIJX5XqtQk_11weavepy_jit2ir5TFuncECs5bNXvwCVaF0_10weavepy_vm
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel10make_dumps0B7_
+__RNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11try_reserveCs5bNXvwCVaF0_10weavepy_vm
+__RNvMs0_NtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder4save
+__RNvMs0_NtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder7memoize
+__RNvMs0_NtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder10save_items
+__RNvMNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel6encodeNtB2_6Framer4data
+__RNvMs0_NtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder7integer
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel6encode15InstanceContextEEB15_
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel10make_loads0B7_
+__RNvMs4_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB5_14UnpicklerGuard5holds
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14int_bit_lengthINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14int_bit_length
+__RNvMsn_NtCshJldXA6Gmzv_10num_bigint6bigintNtB5_6BigInt3pow
+__RNvXs8_NtNtCshJldXA6Gmzv_10num_bigint7biguint14multiplicationRNtB7_7BigUintNtNtNtCs2NHf1dqD6Ob_4core3ops5arith3Mul3mul
+__RNvNtNtCshJldXA6Gmzv_10num_bigint7biguint14multiplication10scalar_mul
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VechE6resizeCs5bNXvwCVaF0_10weavepy_vm
+__RNvXsa_NtNtCshJldXA6Gmzv_10num_bigint7biguint14multiplicationNtB7_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith9MulAssignRBX_E10mul_assign
+__RNvYNvNvMNtCs5bNXvwCVaF0_10weavepy_vm13builtin_typesNtB7_12BuiltinTypes5build9proxy_newINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB9_6object6ObjectEE4callB9_
+__RNvNvMNtCs5bNXvwCVaF0_10weavepy_vm13builtin_typesNtB4_12BuiltinTypes5build9proxy_new
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_chrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_chr
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object16range_len_bigint
+__RNvMss_NtCs5bNXvwCVaF0_10weavepy_vm6objectNtB5_5Range6bounds
+__RNvXs0_NtNtCshJldXA6Gmzv_10num_bigint6bigint11subtractionNtB7_6BigIntINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3SubRBT_E3sub
+__RNvMs1_NtCswHAcCYaqJh_11parking_lot9raw_mutexNtB5_8RawMutex11unlock_slow
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real20join_wait_collecting
+__RNvCs5bNXvwCVaF0_10weavepy_vm11range_slice
+__RNCNvCs5bNXvwCVaF0_10weavepy_vm11range_slice0B3_
+__RNvXNtNtCshJldXA6Gmzv_10num_bigint6bigint11subtractionRNtB4_6BigIntNtNtNtCs2NHf1dqD6Ob_4core3ops5arith3Sub3sub
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyNtNtCsdCPRsQxKMa_16parking_lot_core11parking_lot10ThreadDataECswHAcCYaqJh_11parking_lot
+__RNCNvCs5bNXvwCVaF0_10weavepy_vm11range_slices_0B3_
+__RNvXsf_NtNtCshJldXA6Gmzv_10num_bigint6bigint14multiplicationRNtB7_6BigIntNtNtNtCs2NHf1dqD6Ob_4core3ops5arith3Mul3mul
+__RNvXs_NtNtCshJldXA6Gmzv_10num_bigint6bigint8additionRNtB6_6BigIntINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3AddBP_E3add
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier217wpjit_call_method
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter15do_min_max_call
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter10op_compare
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter21deep_order_collection
+__RNvCs5bNXvwCVaF0_10weavepy_vm12is_nan_value
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier218wpjit_dyn_attr_get
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7sre_mod11sre_compile
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7sre_mod14codeseq_to_vec0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7sre_mod8sre_exec
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib7sre_mod14subject_to_vec
+__RNvMs0_NtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder13save_instance
+__RNCNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel10make_dumps00B9_
+__RNvMs1_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB5_11ImportGuard7modules
+__RNvMs0_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB5_12BuiltinGuard5holds
+__RNvMs_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accelNtB4_20ModuleFunctionsGuard5holds
+__RNvNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel7classes12plain_layout
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCs5bNXvwCVaF0_10weavepy_vm5tier27CallCtxEEB1e_
+__RNvNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel7classes16benign_metaclass
+__RNvNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel7classes14resolve_global
+__RNvNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel7classes10slot_names
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier221finish_deopted_callee
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel5build
+__RNvNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel7classes14is_member_slot
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel1b
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel6encode9ClassPlanE9drop_slowBO_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier213rebuild_stack
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface19install_type_checks18type_instancecheck
+__RNCNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib12pickle_accel10make_loads00B9_
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCs5bNXvwCVaF0_10weavepy_vm5tier28JitStateEE4withNCNvB1x_16try_enter_resume0INtNtBZ_6option6OptionNtB1x_13CompiledEntryEEB1z_
+__RNvCs5bNXvwCVaF0_10weavepy_vm15exception_value
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object10repr_enter
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object10repr_leave
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5tier212NativeCalleeEBF_
+__RNvXsc_NtNtCshJldXA6Gmzv_10num_bigint6bigint11subtractionNtB7_6BigIntINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3SublE3sub
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier218finish_interp_call
+__RNvXs2_NtNtCshJldXA6Gmzv_10num_bigint7biguint11subtractionNtB7_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith9SubAssignmE10sub_assign
+__RNvNtNtCshJldXA6Gmzv_10num_bigint7biguint11subtraction4sub2
+__RNvXsF_NtNtCshJldXA6Gmzv_10num_bigint6bigint8divisionNtB7_6BigIntNtNtNtCs2NHf1dqD6Ob_4core3ops5arith3Div3div
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtNtCs5bNXvwCVaF0_10weavepy_vm5tier211DirectEntryE9drop_slowBH_
+__RNvNtNtCshJldXA6Gmzv_10num_bigint7biguint8division11div_rem_ref
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit5value7JitTypeEENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertB1r_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4ZIJX5XqtQk_11weavepy_jit5value7JitTypeEEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB1u_
+__RNCNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface21install_value_richcmp15richcmp_builtin0B7_
+__RNvNvNtCs5bNXvwCVaF0_10weavepy_vm12type_surface21install_value_richcmp7fam_int
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins13dict_fromkeysINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins13dict_fromkeys
+__RNvXNtNtCshJldXA6Gmzv_10num_bigint7biguint11subtractionNtB4_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3SubRBR_E3sub
+__RNvXs_NtNtCshJldXA6Gmzv_10num_bigint7biguint11subtractionNtB6_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith9SubAssignRBT_E10sub_assign
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11list_removeINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter17make_seq_iterator
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11list_remove
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object9member_eq
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable7ipnsortTjjNtNtCs7oIyYg99ywV_16cranelift_module6module6FuncIdENCINvMB6_SBT_20sort_unstable_by_keyjNCNvMs0_NtCs3j6f2J4rC56_13cranelift_jit7backendNtB2t_9JITModule20finalize_definitionss0_0E0EB2v_
+__RNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze17unary_result_type
+__RNCNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze15analyze_for_rets2_0B5_
+__RNCNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze15analyze_for_rets1_0B5_
+__RNCNvNtCs4ZIJX5XqtQk_11weavepy_jit7analyze15analyze_for_rets8_0B5_
+__RNvCs5bNXvwCVaF0_10weavepy_vm8unary_op
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier212wpjit_str_eq
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier213wpjit_str_get
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier215wpjit_const_str
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier218wpjit_dyn_attr_set
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11set_discardINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11set_discard
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier221native_exit_writeback
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real25make_thread_handle_object0B7_
+__RNvMsf_NtCs5bNXvwCVaF0_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE17try_borrow_lockedB7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real19is_main_interpreter
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared5pivot11median3_recjNvYjNtNtBa_3cmp10PartialOrd2ltECs4ZIJX5XqtQk_11weavepy_jit
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real25make_thread_handle_objects0_0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib11thread_real15thread_shutdown
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm5tier216probe_ctor_field
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs5bNXvwCVaF0_10weavepy_vm5tier214ClassCtorEntryEBF_
+__RNvMs1_NtNtCs6KxzipSu1yP_3std6thread9lifecycleINtB5_9JoinInneruE4joinCs5bNXvwCVaF0_10weavepy_vm
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtNtCs6KxzipSu1yP_3std6thread9lifecycle6PacketuEE9drop_slowCs5bNXvwCVaF0_10weavepy_vm
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins21bytes_from_source_obj0B5_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18bytearray_byte_arg
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10bytes_findINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins10bytes_find
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins20bytes_needle_guarded
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object24bytearray_export_acquire
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm6object24bytearray_export_release
+__RNvNtCshubbc774ReN_6memchr6memmem4find
+__RINvMNtNtCshubbc774ReN_6memchr6memmem8searcherNtB3_8Searcher3newNtNtNtNtB7_4arch3all10packedpair20DefaultFrequencyRankECs5bNXvwCVaF0_10weavepy_vm
+__RNvNtNtCshubbc774ReN_6memchr6memmem8searcher22searcher_kind_one_byte
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19dispatch_inplace_op
+__RNCNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins14numeric_dunders2_0B5_
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter13do_float_call
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins7b_float
+__RNCINvNtCs5bNXvwCVaF0_10weavepy_vm8builtins9method_kwNvB4_18bytes_translate_kwEs_0B6_
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins18bytes_translate_kwINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1I_EEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins13bytes_argview
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11list_insertINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins11list_insert
+__RNvYNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_strINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCs5bNXvwCVaF0_10weavepy_vm8builtins5b_str
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter19format_template_str
+__RNvCs5bNXvwCVaF0_10weavepy_vm17scan_format_field
+__RNvCs5bNXvwCVaF0_10weavepy_vm18split_format_field
+__RNvCs5bNXvwCVaF0_10weavepy_vm19split_name_trailers
+__RNvCs5bNXvwCVaF0_10weavepy_vm14format_int_hex
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel17json_make_scanner
+__RNvMs3_Cs5bNXvwCVaF0_10weavepy_vmNtB5_11Interpreter8op_truth
+__RNvCs5bNXvwCVaF0_10weavepy_vm18long_overflows_f64
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel5builds_0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel17json_make_encoder
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs5bNXvwCVaF0_10weavepy_vm6object6ObjectEE6resizeB1l_
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel17json_make_encoders5_0B7_
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel12encode_value
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel17encode_value_impl
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel9json_kind
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accelNtB5_10JsonWriter8push_str
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel20write_encoded_string
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accelNtB5_10JsonWriter11quoted_utf8
+__RNvMs3_NtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accelNtB5_10JsonWriter11push_object
+__RNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel11encode_list0B7_
+__RNvXs_NtCsbLzj8cKgNSH_4zmij7privatedNtB4_6Sealed20write_to_zmij_buffer
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel17json_make_encoders5_0EBJ_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel10EncoderCfgE9drop_slowBM_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtB7_3vec3VecmEE9drop_slowCs5bNXvwCVaF0_10weavepy_vm
+__RNCNCNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel17json_make_scanners_00B9_
+__RINvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel9scan_oncehEB6_
+__RINvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel12parse_objecthEB6_
+__RINvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel21scan_string_with_memohEB6_
+__RNvXNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accelhNtB2_8JsonChar3key
+__RINvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel11parse_arrayhEB6_
+__RNvXNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accelhNtB2_8JsonChar6string
+__RNvNtNtCs5bNXvwCVaF0_10weavepy_vm6stdlib10json_accel13finish_object
+__RNvNtCskHi7NRqR3VY_10weavepy_vm9proc_init16snapshot_std_fds
+__RNvCsdSCkLiPtoQj_11weavepy_cli8cli_main
+__RNvNtCskHi7NRqR3VY_10weavepy_vm9proc_init23restore_initial_std_fds
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod34block_async_signals_current_thread
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod12StoredSigsetE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEBZ_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod12StoredSigsetE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B1M_
+__RNSNvYNCINvNtNtCs6KxzipSu1yP_3std6thread9lifecycle15spawn_uncheckedNCNvCsdSCkLiPtoQj_11weavepy_cli18run_on_large_stack0lEs_0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceuE9call_once6vtableB17_
+__RINvNtNtCs6KxzipSu1yP_3std3sys9backtrace28___rust_begin_short_backtraceNCNCNCINvNtNtB6_6thread9lifecycle15spawn_uncheckedNCNvCsdSCkLiPtoQj_11weavepy_cli18run_on_large_stack0lEs_000uEB1Z_
+__RINvNtNtCs6KxzipSu1yP_3std3sys9backtrace28___rust_begin_short_backtraceNCNvCsdSCkLiPtoQj_11weavepy_cli18run_on_large_stack0lEB1b_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod36unblock_async_signals_current_thread
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod28install_startup_dispositions
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtNtBa_6poison5mutex5MutexINtNtNtNtBc_11collections4hash3map7HashMaplNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEE10initializeNCINvB1a_11get_or_initNCNvNtNtB2U_6stdlib10signal_mod8handlers0E0zE0E0B2U_
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell4CellTyyEEE4withNCNvMNtNtBa_4hash6randomNtB1I_11RandomState3new0B21_ECskHi7NRqR3VY_10weavepy_vm
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvNtNtB1K_6stdlib10signal_mod23default_int_handler_obj0E0zE0E0B1K_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod1b
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMaplNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBS_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRlECskHi7NRqR3VY_10weavepy_vm
+__RNvXs3_NtNtCs2NHf1dqD6Ob_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCskHi7NRqR3VY_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTlNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherlBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod11set_handler
+__RINvMNtNtNtCs6KxzipSu1yP_3std3sys4sync8once_boxINtB3_7OnceBoxNtNtNtNtNtB7_3pal4unix4sync5mutex5MutexE10initializeNCNvMNtNtB5_5mutex7pthreadNtB1S_5Mutex3get0ECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEBF_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod18set_os_disposition
+__RNvCsdSCkLiPtoQj_11weavepy_cli13main_dispatch
+__RNvMNtNtNtCsk4JIIOXCTm1_18tracing_subscriber6filter3env7builderNtB2_7Builder12try_from_env
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtNtCsk4JIIOXCTm1_18tracing_subscriber6filter3env7builder7BuilderECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsdq5mISKDh0s_5alloc6string6StringECsdSCkLiPtoQj_11weavepy_cli
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE10initializeNCINvB2_11get_or_initNCNvNtNtBX_6stdlib10signal_mod23default_int_handler_obj0E0zEBX_
+__RNvMNtNtNtCsk4JIIOXCTm1_18tracing_subscriber6filter3env9directiveNtB2_9Directive5parse
+__RNvXse_NtCslkUo1JD93Bq_12tracing_core8metadataNtB5_11LevelFilterNtNtNtCs2NHf1dqD6Ob_4core3str6traits7FromStr8from_str
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtNtCsk4JIIOXCTm1_18tracing_subscriber6filter3env9directive9DirectiveECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCsk4JIIOXCTm1_18tracing_subscriber6filter3env12FromEnvErrorECsdSCkLiPtoQj_11weavepy_cli
+__RNvXsb_NtNtNtCs6KxzipSu1yP_3std4sync6poison6rwlockINtB5_6RwLockINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslkUo1JD93Bq_12tracing_core10dispatcher9RegistrarEENtNtCs2NHf1dqD6Ob_4core7default7Default7defaultB1A_
+__RNvCsdSCkLiPtoQj_11weavepy_cli12bridged_args
+__RINvMNtNtNtCs6KxzipSu1yP_3std3sys4sync8once_boxINtB3_7OnceBoxNtNtNtNtNtB7_3pal4unix4sync5mutex5MutexE10initializeNCNvMNtNtB5_5mutex7pthreadNtB1S_5Mutex3get0ECsdSCkLiPtoQj_11weavepy_cli
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtNtCsk4JIIOXCTm1_18tracing_subscriber6filter3env9directive9DirectiveE8grow_oneBU_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsk4JIIOXCTm1_18tracing_subscriber
+__RNvMNtNtNtCsk4JIIOXCTm1_18tracing_subscriber6filter3env9directiveNtB2_9Directive9to_static
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtNtCsk4JIIOXCTm1_18tracing_subscriber6filter3env5field5MatchEECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtNtCsk4JIIOXCTm1_18tracing_subscriber6filter3env9directive9DirectiveEECsdSCkLiPtoQj_11weavepy_cli
+__RINvMs2_NtCs4my1BO6s0Jb_9once_cell3impINtB6_8OnceCellINtNtNtNtCs6KxzipSu1yP_3std4sync6poison6rwlock6RwLockINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslkUo1JD93Bq_12tracing_core10dispatcher9RegistrarEEE10initializeNCINvMs4_NtB8_4syncINtB3t_8OnceCellBP_E11get_or_initNCNvMs9_B3t_INtB3t_4LazyBP_E5force0E0NtNvMs4_B3t_IB3E_pE11get_or_init4VoidEB2h_
+__RNvNtCs4my1BO6s0Jb_9once_cell3imp18initialize_or_wait
+__RNCINvMs2_NtCs4my1BO6s0Jb_9once_cell3impINtB8_8OnceCellINtNtNtNtCs6KxzipSu1yP_3std4sync6poison6rwlock6RwLockINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslkUo1JD93Bq_12tracing_core10dispatcher9RegistrarEEE10initializeNCINvMs4_NtBa_4syncINtB3v_8OnceCellBR_E11get_or_initNCNvMs9_B3v_INtB3v_4LazyBR_E5force0E0NtNvMs4_B3v_IB3G_pE11get_or_init4VoidE0B2j_
+__RNvXs3_NtCs4my1BO6s0Jb_9once_cell3impNtB5_5GuardNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCslkUo1JD93Bq_12tracing_core10dispatcher9RegistrarE8grow_oneBQ_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner14grow_amortizedCslkUo1JD93Bq_12tracing_core
+__RNvYINtNtCsk4JIIOXCTm1_18tracing_subscriber3fmt10SubscriberNtNtB5_6format13DefaultFieldsNtBX_6FormatNtNtNtB7_6filter3env9EnvFilterENtNtCslkUo1JD93Bq_12tracing_core10subscriber10Subscriber20on_register_dispatchCsdSCkLiPtoQj_11weavepy_cli
+__RNvXs0_NtCsk4JIIOXCTm1_18tracing_subscriber3fmtINtB5_10SubscriberNtNtB5_6format13DefaultFieldsNtB13_6FormatNtNtNtB7_6filter3env9EnvFilterENtNtCslkUo1JD93Bq_12tracing_core10subscriber10Subscriber14max_level_hintCsdSCkLiPtoQj_11weavepy_cli
+__RNvMs1_NtCs9rf1yDsT1Nb_11tracing_log10log_tracerNtB5_7Builder4init
+__RNvNvCskHi7NRqR3VY_10weavepy_vm15os_args_bridged21register_if_ambiguous
+__RNvCsdSCkLiPtoQj_11weavepy_cli9real_main
+__RNvXs_NtNtCsdq5mISKDh0s_5alloc3vec14spec_from_iterINtB6_3VecNtNtB8_6string6StringEINtB4_12SpecFromIterBW_INtNtB6_9into_iter8IntoIterBW_EE9from_iterCsdSCkLiPtoQj_11weavepy_cli
+__RNvXs0_NtNtCsdq5mISKDh0s_5alloc3vec11spec_extendINtB7_3VecNtNtB9_6string6StringEINtB5_10SpecExtendBU_INtNtB7_9into_iter8IntoIterBU_EE11spec_extendCs9DEk322Hfll_16weavepy_compiler
+__RNvXs3_CsdSCkLiPtoQj_11weavepy_cliNtB5_3CliNtNtCs8ktk9OOIm5l_12clap_builder6derive4Args12augment_args
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs8ktk9OOIm5l_12clap_builder4util2id2IdE8grow_oneBS_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs8ktk9OOIm5l_12clap_builder
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs8ktk9OOIm5l_12clap_builder7builder9arg_group8ArgGroupE8grow_oneBS_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtCs8ktk9OOIm5l_12clap_builder7builder12value_parser11ValueParserEECsdSCkLiPtoQj_11weavepy_cli
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs8ktk9OOIm5l_12clap_builder
+__RNvMs4_NtCs8ktk9OOIm5l_12clap_builder7mkeymapNtB5_7MKeyMap4push
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs8ktk9OOIm5l_12clap_builder7builder3arg3ArgE8grow_oneBS_
+__RINvMs_NtNtCs8ktk9OOIm5l_12clap_builder7builder3argNtB5_3Arg12value_parserINtNtB7_12value_parser20RangedI64ValueParserhEECsdSCkLiPtoQj_11weavepy_cli
+__RINvMs0_NtNtCs8ktk9OOIm5l_12clap_builder7builder7commandNtB6_7Command5aboutReECsdSCkLiPtoQj_11weavepy_cli
+__RINvMs0_NtNtCs8ktk9OOIm5l_12clap_builder7builder7commandNtB6_7Command10long_aboutReECsdSCkLiPtoQj_11weavepy_cli
+__RINvMNtNtCs8ktk9OOIm5l_12clap_builder7builder7commandNtB3_7Command16get_matches_fromINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB1p_6string6StringEB1R_ECsdSCkLiPtoQj_11weavepy_cli
+__RNvMs3_NtNtCs8ktk9OOIm5l_12clap_builder7builder7commandNtB5_7Command9__do_parse
+__RNvMs3_NtNtCs8ktk9OOIm5l_12clap_builder7builder7commandNtB5_7Command11__build_self
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs8ktk9OOIm5l_12clap_builder7builder7command7CommandEEB1e_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs8ktk9OOIm5l_12clap_builder7mkeymap3KeyE8grow_oneBQ_
+__RNvMs_NtNtCs8ktk9OOIm5l_12clap_builder6parser6parserNtB4_6Parser16get_matches_with
+__RNvMs_NtNtCs8ktk9OOIm5l_12clap_builder6parser6parserNtB4_6Parser15resolve_pending
+__RNvMs_NtNtCs8ktk9OOIm5l_12clap_builder6parser6parserNtB4_6Parser5react
+__RNvMNtNtCs8ktk9OOIm5l_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher6remove
+__RNvMs_NtNtCs8ktk9OOIm5l_12clap_builder6parser6parserNtB4_6Parser16start_custom_arg
+__RNvXsc_NtNtCs8ktk9OOIm5l_12clap_builder7builder12value_parserNtB5_15BoolValueParserNtB5_14AnyValueParser7type_idB9_
+__RNvMs0_NtNtCs8ktk9OOIm5l_12clap_builder4util8flat_mapINtB5_5EntryNtNtB7_2id2IdNtNtNtNtB9_6parser7matches11matched_arg10MatchedArgE9or_insertB9_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtNtCs8ktk9OOIm5l_12clap_builder6parser7matches11matched_arg10MatchedArgE8grow_oneBU_
+__RNvMNtNtNtCs8ktk9OOIm5l_12clap_builder6parser7matches11matched_argNtB2_10MatchedArg13new_val_group
+__RNvXNtCsiNeM0PM367B_8clap_lex3extNtNtNtCs6KxzipSu1yP_3std3ffi6os_str5OsStrNtB2_8OsStrExt12strip_prefix
+__RNvMs1_CsiNeM0PM367B_8clap_lexNtB5_9ParsedArg8to_short
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecNtNtNtCs8ktk9OOIm5l_12clap_builder4util9any_value8AnyValueEE8grow_oneB18_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecNtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringEE8grow_oneCs8ktk9OOIm5l_12clap_builder
+__RNvMs_NtNtCs8ktk9OOIm5l_12clap_builder6parser6parserNtB4_6Parser15push_arg_values
+__RNvXsC_NtNtCs8ktk9OOIm5l_12clap_builder7builder12value_parserNtB5_15BoolValueParserNtB5_16TypedValueParser9parse_ref
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs8ktk9OOIm5l_12clap_builder4util9any_value8AnyValueE8grow_oneBS_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecjE8grow_oneCs8ktk9OOIm5l_12clap_builder
+__RNvMs_NtNtCs8ktk9OOIm5l_12clap_builder6parser6parserNtB4_6Parser12add_defaults
+__RNvXsc_NtNtCs8ktk9OOIm5l_12clap_builder7builder12value_parserNtB5_15BoolValueParserNtB5_14AnyValueParser10parse_ref_B9_
+__RNvMNtNtCs8ktk9OOIm5l_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher10add_val_to
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs8ktk9OOIm5l_12clap_builder6parser6parser11ParseResultEBH_
+__RNvXsc_NtNtCs8ktk9OOIm5l_12clap_builder7builder12value_parserINtB5_20RangedI64ValueParserhENtB5_14AnyValueParser7type_idCsdSCkLiPtoQj_11weavepy_cli
+__RNvXsu_NtNtCs8ktk9OOIm5l_12clap_builder7builder12value_parserINtB5_20RangedI64ValueParserhENtB5_16TypedValueParser9parse_refCsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCs8ktk9OOIm5l_12clap_builder4util5graph10ChildGraphNtNtBG_2id2IdEEBI_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtNtCs8ktk9OOIm5l_12clap_builder6parser7matches11matched_arg10MatchedArgEECsdSCkLiPtoQj_11weavepy_cli
+__RNvMNtNtCs8ktk9OOIm5l_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher10into_inner
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtCs8ktk9OOIm5l_12clap_builder6parser6parser10PendingArgEEB13_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs8ktk9OOIm5l_12clap_builder7builder7command7CommandECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs8ktk9OOIm5l_12clap_builder7builder3arg3ArgECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtNtCs8ktk9OOIm5l_12clap_builder4util2id2IdNtNtNtB1f_7builder13arg_predicate12ArgPredicateINtNtB4_6option6OptionIBC_NtNtB1W_6os_str5OsStrEEEEECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs8ktk9OOIm5l_12clap_builder7builder3ext10ExtensionsECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs8ktk9OOIm5l_12clap_builder4util9any_value8AnyValueEECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs8ktk9OOIm5l_12clap_builder7builder7command7CommandEECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs8ktk9OOIm5l_12clap_builder7builder9arg_group8ArgGroupEECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs8ktk9OOIm5l_12clap_builder7builder9arg_group8ArgGroupECsdSCkLiPtoQj_11weavepy_cli
+__RNvXs2_CsdSCkLiPtoQj_11weavepy_cliNtB5_3CliNtNtCs8ktk9OOIm5l_12clap_builder6derive14FromArgMatches20from_arg_matches_mut
+__RINvMNtNtCs8ktk9OOIm5l_12clap_builder4util8flat_mapINtB3_7FlatMapNtNtB5_2id2IdNtNtNtNtB7_6parser7matches11matched_arg10MatchedArgE12remove_entryeECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecIBC_NtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringEEECs8ktk9OOIm5l_12clap_builder
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringEECs8ktk9OOIm5l_12clap_builder
+__RNvXNtCs2NHf1dqD6Ob_4core3anybNtB2_3Any7type_idCs8ktk9OOIm5l_12clap_builder
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtNtB4_4iter8adapters7flatten7FlattenINtNtNtCsdq5mISKDh0s_5alloc3vec9into_iter8IntoIterINtB1m_3VecNtNtNtCs8ktk9OOIm5l_12clap_builder4util9any_value8AnyValueEEEECsdSCkLiPtoQj_11weavepy_cli
+__RNvXNtCs2NHf1dqD6Ob_4core3anyhNtB2_3Any7type_idCsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtNtCs8ktk9OOIm5l_12clap_builder6parser7matches11arg_matches10ArgMatchesECsdSCkLiPtoQj_11weavepy_cli
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons13raw_arg_bytes
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins16bridge_to_object
+__RINvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB7_6Object8from_strReEB9_
+__RNvXsz_NtCskHi7NRqR3VY_10weavepy_vm12shared_valueNtB5_9SharedStrINtNtCs2NHf1dqD6Ob_4core7convert4FromReE4from
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VechEECskHi7NRqR3VY_10weavepy_vm
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECsdSCkLiPtoQj_11weavepy_cli
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsdSCkLiPtoQj_11weavepy_cli
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCskHi7NRqR3VY_10weavepy_vm12shared_value12SliceStoragehEE9drop_slowBL_
+__RNvCsdSCkLiPtoQj_11weavepy_cli11build_flags
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBG_6string6StringEECsdSCkLiPtoQj_11weavepy_cli
+__RNvXsL_CskHi7NRqR3VY_10weavepy_vmNtB5_16InterpreterFlagsNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvCsdSCkLiPtoQj_11weavepy_cli23run_source_with_options
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockFEuE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zECskHi7NRqR3VY_10weavepy_vm
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockFEuE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0CskHi7NRqR3VY_10weavepy_vm
+__RNvMCsdSCkLiPtoQj_11weavepy_cliNtB2_12EnvOverrides8from_env
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockFG_RL0_eENtNtCslXHHUCZmUMI_14weavepy_parser6parser21UnicodeNameResolutionE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEB16_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockFG_RL0_eENtNtCslXHHUCZmUMI_14weavepy_parser6parser21UnicodeNameResolutionE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B1T_
+__RNvNtCs67nvpeQgtOq_13weavepy_lexer7scanner16tokenize_partial
+__RNvMNtCs67nvpeQgtOq_13weavepy_lexer7scannerNtB2_7Scanner29scan_ident_or_prefixed_string
+__RNvMs0_NtCs67nvpeQgtOq_13weavepy_lexer5tokenNtB5_7Keyword10from_ident
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs67nvpeQgtOq_13weavepy_lexer5token5TokenE8grow_oneBQ_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs67nvpeQgtOq_13weavepy_lexer
+__RNvXsc_NtNtCs8ktk9OOIm5l_12clap_builder7builder12value_parserINtB5_20RangedI64ValueParserhENtB5_14AnyValueParser10parse_ref_CsdSCkLiPtoQj_11weavepy_cli
+__RNvMs5_NtNtCs8ktk9OOIm5l_12clap_builder7builder7commandNtB5_7Command14required_graph
+__RNvMNtNtCs8ktk9OOIm5l_12clap_builder6parser9validatorNtB2_9Validator8validate
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCs8ktk9OOIm5l_12clap_builder4util8flat_map7FlatMapNtNtBG_2id2IdINtNtCsdq5mISKDh0s_5alloc3vec3VecB1w_EEEBI_
+__RNvMNtNtCs8ktk9OOIm5l_12clap_builder6parser11arg_matcherNtB2_10ArgMatcher21fill_in_global_values
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCs8ktk9OOIm5l_12clap_builder4util8flat_map7FlatMapNtNtBG_2id2IdNtNtNtNtBI_6parser7matches11matched_arg10MatchedArgEEBI_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs6KxzipSu1yP_3std4path7PathBufEECsdSCkLiPtoQj_11weavepy_cli
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once9call_onceNCNvNtCsbBbqfalDtZG_12weavepy_capi6loader27install_vm_extension_loader0E0B14_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10ext_loader24install_extension_loader
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECskHi7NRqR3VY_10weavepy_vm
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecmE8grow_oneCs67nvpeQgtOq_13weavepy_lexer
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13codecs_engine11utf8_encode
+__RNvMs_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13codecs_engineNtB4_6EncCtx3new
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13codecs_engine6EncCtxEBH_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10codecs_mod12encoding_key
+__RNvMs_NtNtCs8ktk9OOIm5l_12clap_builder6parser6parserNtB4_6Parser19possible_subcommand
+__RNvCskHi7NRqR3VY_10weavepy_vm17bridged_arg_bytes
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10codecs_mod10encode_obj
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10codecs_mod10encode_str
+__RNvCsdSCkLiPtoQj_11weavepy_cli8run_path
+__RNvCskHi7NRqR3VY_10weavepy_vm25decode_source_bytes_inner
+__RNvCskHi7NRqR3VY_10weavepy_vm25translate_source_newlines
+__RNvMNtCs67nvpeQgtOq_13weavepy_lexer7scannerNtB2_7Scanner11scan_string
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecThjEE8grow_oneCs67nvpeQgtOq_13weavepy_lexer
+__RNvMNtCs67nvpeQgtOq_13weavepy_lexer7scannerNtB2_7Scanner20verify_end_of_number
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser25parse_with_flufl_tracking
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser12parse_module
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser24skip_trivia_and_newlines
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser15parse_statement
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser24check_stmt_as_ifexp_body
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser16consume_stmt_end
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCslXHHUCZmUMI_14weavepy_parser3ast4StmtE8grow_oneBQ_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCslXHHUCZmUMI_14weavepy_parser
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser12apply_poison
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser4meta
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser6parser6ParserEBF_
+__RNvNvCs9DEk322Hfll_16weavepy_compiler22collect_module_imports4walk
+__RNvNvCs9DEk322Hfll_16weavepy_compiler21module_mentions_super7in_stmt
+__RINvXNvMNtCsdq5mISKDh0s_5alloc5sliceSp9to_vec_inNtNtCslXHHUCZmUMI_14weavepy_parser3ast4StmtNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs9DEk322Hfll_16weavepy_compiler
+__RNvNtCs9DEk322Hfll_16weavepy_compiler7ast_opt9fold_stmt
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler19compile_module_body
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler4emit
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs9DEk322Hfll_16weavepy_compiler
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler15resolve_colspan
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler12compile_stmt
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBG_6string6StringEECs9DEk322Hfll_16weavepy_compiler
+__RINvMNtCs9DEk322Hfll_16weavepy_compiler6internNtB3_11InternIndex6internNtB5_8ConstantNvB3_13constant_hashEB5_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCs9DEk322Hfll_16weavepy_compiler8ConstantE8grow_oneBO_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecjE8grow_oneCs9DEk322Hfll_16weavepy_compiler
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs9DEk322Hfll_16weavepy_compiler9flowgraph5BlockE8grow_oneBQ_
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs9DEk322Hfll_16weavepy_compiler
+__RNvNtCs9DEk322Hfll_16weavepy_compiler12cpython_code14map_to_cpython
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9DEk322Hfll_16weavepy_compiler6intern11InternIndexEBF_
+__RNvNtCs9DEk322Hfll_16weavepy_compiler8validate15validate_module
+__RNvMNtCs9DEk322Hfll_16weavepy_compiler8validateNtB2_5Scope3new
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9DEk322Hfll_16weavepy_compiler8validate9ValidatorEBF_
+__RNvCs9DEk322Hfll_16weavepy_compiler25collect_global_decls_deep
+__RNvCs9DEk322Hfll_16weavepy_compiler19for_each_scope_expr
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs9DEk322Hfll_16weavepy_compiler8bytecode11InstructionE8grow_oneBQ_
+__RNvXs1z_Cs9DEk322Hfll_16weavepy_compilerNtB6_8ConstantNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg20optimize_basic_block
+__RNvXs3_NtNtCs2NHf1dqD6Ob_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCs9DEk322Hfll_16weavepy_compiler
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg31remove_redundant_nops_and_jumps
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg5shape
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtBU_3Cfg18optimize_load_fast3RefE8grow_oneBW_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCs2DfPjnIE6AW_8indexmap3map8IndexMapNtNtCsdq5mISKDh0s_5alloc6string6StringNtCs9DEk322Hfll_16weavepy_compiler7BindingEEB1U_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCs9DEk322Hfll_16weavepy_compiler9LoopFrameEEB1a_
+__RNvMs9_NtCs9DEk322Hfll_16weavepy_compiler8bytecodeNtB5_10CacheTable6resize
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCsdq5mISKDh0s_5alloc6string6StringuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs9DEk322Hfll_16weavepy_compiler
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtCs9DEk322Hfll_16weavepy_compiler9LineIndexE9drop_slowBF_
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtNtNtCs6KxzipSu1yP_3std11collections4hash3set7HashSetNtNtB7_6string6StringEE9drop_slowCs9DEk322Hfll_16weavepy_compiler
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RceE9drop_slowCs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast4StmtEECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast8StmtKindECs9DEk322Hfll_16weavepy_compiler
+__RNvCskHi7NRqR3VY_10weavepy_vm18argv_str_to_object
+__RNvXsH_NtCskHi7NRqR3VY_10weavepy_vm12shared_valueNtB5_9SharedStrNtNtCs2NHf1dqD6Ob_4core3fmt7Display3fmt
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser22parse_subscript_single
+__RNvCs9DEk322Hfll_16weavepy_compiler43should_apply_two_element_slice_optimization
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler17starunpack_helper
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler19compile_load_target
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser11parse_while
+__RNvMsE_Cs9DEk322Hfll_16weavepy_compilerNtB5_10CodeObject10content_eq
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast5AliasEECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast3ArgEECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast9ArgumentsECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast3ArgECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCslXHHUCZmUMI_14weavepy_parser3ast4ExprEEECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast4ExprEECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs9DEk322Hfll_16weavepy_compiler8validate9DirectiveEEB1c_
+__RNvMsJ_Cs9DEk322Hfll_16weavepy_compilerNtB5_9LineIndex3new
+__RNvCs9DEk322Hfll_16weavepy_compiler16collect_assigned
+__RNvCs9DEk322Hfll_16weavepy_compiler19collect_walrus_stmt
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler19register_comp_cells
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCs9DEk322Hfll_16weavepy_compiler7ColSpanE8grow_oneBO_
+__RNvCs9DEk322Hfll_16weavepy_compiler21block_has_annotations
+__RNvCs9DEk322Hfll_16weavepy_compiler30block_has_deferred_annotations
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler18compile_stmt_inner
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler6finish
+__RNvNtCs9DEk322Hfll_16weavepy_compiler9flowgraph8optimize
+__RNvNtCs9DEk322Hfll_16weavepy_compiler12cpython_code13stack_effects
+__RNvMs0_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg5chain
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg18remove_unreachable
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg20resolve_line_numbers
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCs9DEk322Hfll_16weavepy_compiler8ConstantEBD_
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg27block_remove_redundant_nops
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCs9DEk322Hfll_16weavepy_compiler8ConstantEEB1a_
+__RNvCs9DEk322Hfll_16weavepy_compiler11make_params
+__RNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB4_9Validator10visit_stmt
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9DEk322Hfll_16weavepy_compiler8validate5ScopeEBF_
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler3new
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCs9DEk322Hfll_16weavepy_compiler12FinallyFrameEEB1a_
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg18optimize_load_fast
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs9DEk322Hfll_16weavepy_compiler9flowgraph5BlockEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionTNtNtCsdq5mISKDh0s_5alloc6string6StringBY_BY_EEECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCs9DEk322Hfll_16weavepy_compiler18DeferredAnnotationEEB1a_
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser19find_invalid_target
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser16parse_assign_rhs
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser12parse_number
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser22expect_or_forgot_comma
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser26take_trailing_type_comment
+__RNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB4_9Validator12visit_target
+__RNvCs9DEk322Hfll_16weavepy_compiler20collect_target_names
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB1m_NtNtBa_3cmp10PartialOrd2ltECs9DEk322Hfll_16weavepy_compiler
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapNtNtCsdq5mISKDh0s_5alloc6string6StringuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE12contains_keyBO_ECs9DEk322Hfll_16weavepy_compiler
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCslXHHUCZmUMI_14weavepy_parser3ast5CmpOpE8grow_oneBQ_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBG_6string6StringEECslXHHUCZmUMI_14weavepy_parser
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtCsdq5mISKDh0s_5alloc6string6StringECs9DEk322Hfll_16weavepy_compiler
+__RNvNvCs9DEk322Hfll_16weavepy_compiler21module_mentions_super7in_expr
+__RNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB4_9Validator22push_type_params_scope
+__RNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB4_9Validator14visit_function
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTReNtNtCs67nvpeQgtOq_13weavepy_lexer5token4SpanEE8grow_oneCs9DEk322Hfll_16weavepy_compiler
+__RNvCs9DEk322Hfll_16weavepy_compiler19collect_walrus_expr
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler25compile_default_arguments
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler18emit_make_function
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast8ExprKindECs9DEk322Hfll_16weavepy_compiler
+__RNvCs9DEk322Hfll_16weavepy_compiler27collect_reads_assign_target
+__RNvXs1b_NtCslXHHUCZmUMI_14weavepy_parser3astNtB6_8ConstantNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvXsG_Cs9DEk322Hfll_16weavepy_compilerNtB5_8ConstantINtNtCs2NHf1dqD6Ob_4core7convert4FromNtNtCslXHHUCZmUMI_14weavepy_parser3ast8ConstantE4from
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler14unwind_fblocks
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler7emit_at
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler18close_unwind_holes
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler22super_attr_optimizable
+__RINvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_expr_inners4_0EB6_
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg20const_loading_instrs
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast8ConstantECs9DEk322Hfll_16weavepy_compiler
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler35import_flag_shadowed_by_module_comp
+__RINvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_expr_inners2_0EB6_
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler15compile_jump_if
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler24genexp_call_optimization
+__RNCNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB7_8Compiler18compile_expr_inners1_0B7_
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler15compile_compare
+__RNvCs9DEk322Hfll_16weavepy_compiler11emit_cmp_op
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler10patch_jump
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRmECs9DEk322Hfll_16weavepy_compiler
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg36is_exit_or_eval_check_without_lineno
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser26parse_target_list_no_tuple
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser20parse_target_or_star
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser9parse_for
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCs9DEk322Hfll_16weavepy_compiler9LoopFrameE8grow_oneBO_
+__RNvMs1_NtCs67nvpeQgtOq_13weavepy_lexer5tokenNtB5_12StringPrefix5parse
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser22parse_simple_statement
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser21parse_expression_list
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser24parse_ternary_or_starred
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser13parse_ternary
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser8parse_or
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser9parse_and
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser9parse_not
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser16parse_comparison
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser12parse_bit_or
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser13parse_bit_xor
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser13parse_bit_and
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser11parse_shift
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser12parse_addsub
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser12parse_muldiv
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser11parse_unary
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser11parse_power
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser10parse_atom
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser13string_prefix
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser6lexeme
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser13decode_string
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser12strip_quotes
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser15decode_str_body
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser15cps_to_constant
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECslXHHUCZmUMI_14weavepy_parser
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser16constant_str_cps
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser24check_string_expr_string
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser19at_expression_start
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser5check
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast4ExprEEB1c_
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser12parse_import
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser17parse_dotted_name
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser6expect
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser5ident
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser3eat
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser24check_import_alias_debug
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCslXHHUCZmUMI_14weavepy_parser3ast5AliasE8grow_oneBQ_
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser18parse_function_def
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser26collect_pep695_type_params
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser21parse_arguments_inner
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser18try_arg_annotation
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCslXHHUCZmUMI_14weavepy_parser3ast3ArgE8grow_oneBQ_
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser11parse_block
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecReE8grow_oneCslXHHUCZmUMI_14weavepy_parser
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCslXHHUCZmUMI_14weavepy_parser3ast4ExprE8grow_oneBQ_
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser23take_colon_type_comment
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser27take_def_body_type_comments
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser12parse_return
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser8parse_if
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser15parse_call_args
+__RNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB4_9Validator10visit_expr
+__RNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB4_9Validator13mark_assigned
+__RNvCs9DEk322Hfll_16weavepy_compiler13collect_decls
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs9DEk322Hfll_16weavepy_compiler8validate5ScopeE8grow_oneBQ_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsdq5mISKDh0s_5alloc6string6StringECs9DEk322Hfll_16weavepy_compiler
+__RNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB4_9Validator10visit_body
+__RNvXsZ_NtCslXHHUCZmUMI_14weavepy_parser3astNtB5_8ExprKindNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast5AliasENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneCs9DEk322Hfll_16weavepy_compiler
+__RNvXsM_NtCslXHHUCZmUMI_14weavepy_parser3astNtB5_9ArgumentsNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RINvXNvMNtCsdq5mISKDh0s_5alloc5sliceSp9to_vec_inNtNtCslXHHUCZmUMI_14weavepy_parser3ast3ArgNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs9DEk322Hfll_16weavepy_compiler
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast9TypeParamENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneCs9DEk322Hfll_16weavepy_compiler
+__RINvXNvMNtCsdq5mISKDh0s_5alloc5sliceSp9to_vec_inNtNtCslXHHUCZmUMI_14weavepy_parser3ast7KeywordNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs9DEk322Hfll_16weavepy_compiler
+__RNvNtCs9DEk322Hfll_16weavepy_compiler7ast_opt9fold_expr
+__RNvMs_NtCs2DfPjnIE6AW_8indexmap5innerINtB4_4CoreNtNtCsdq5mISKDh0s_5alloc6string6StringNtCs9DEk322Hfll_16weavepy_compiler7BindingE10push_entryB1o_
+__RNCNvCs9DEk322Hfll_16weavepy_compiler32collect_comp_walrus_targets_stmt0B3_
+__RNvNvCs9DEk322Hfll_16weavepy_compiler32collect_comp_walrus_targets_stmt4walk
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler23collect_comp_cells_expr
+__RNvCs9DEk322Hfll_16weavepy_compiler15clean_docstring
+__RNvXsF_Cs9DEk322Hfll_16weavepy_compilerNtB5_8ConstantNtNtCs2NHf1dqD6Ob_4core3cmp9PartialEq2eq
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler15emit_store_name
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler18classify_for_store
+__RINvMs3_NtCs2DfPjnIE6AW_8indexmap3mapINtB6_8IndexMapNtNtCsdq5mISKDh0s_5alloc6string6StringNtCs9DEk322Hfll_16weavepy_compiler7BindingE3geteEB1s_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneReECs9DEk322Hfll_16weavepy_compiler
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler26compile_function_def_inner
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler26build_function_object_full
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler19child_qualname_impl
+__RNvCs9DEk322Hfll_16weavepy_compiler19stmt_contains_yield
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler9free_scan
+__RNvCs9DEk322Hfll_16weavepy_compiler19expr_contains_yield
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler22analyze_scope_function
+__RNvCs9DEk322Hfll_16weavepy_compiler21collect_reads_stmt_in
+__RNvCs9DEk322Hfll_16weavepy_compiler26collect_reads_stmt_in_impl
+__RNvCs9DEk322Hfll_16weavepy_compiler18collect_reads_expr
+__RNvCs9DEk322Hfll_16weavepy_compiler18collect_inner_free
+__RNvCs9DEk322Hfll_16weavepy_compiler23collect_inner_free_expr
+__RNvCs9DEk322Hfll_16weavepy_compiler27collect_pep695_header_reads
+__RNvCs9DEk322Hfll_16weavepy_compiler23collect_inner_free_impl
+__RNCNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB7_8Compiler22analyze_scope_functions0_0B7_
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler28collect_uses_through_inlined
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler18compile_expr_inner
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler14emit_load_name
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler16var_index_or_add
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler14compile_assign
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg21scan_block_for_locals
+__RNvNtNtCs8ktk9OOIm5l_12clap_builder6parser9validator23gather_direct_conflicts
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecNtNtNtCs8ktk9OOIm5l_12clap_builder4util2id2IdEE8grow_oneB18_
+__RNvMs_NtNtCs8ktk9OOIm5l_12clap_builder6parser9validatorNtB4_9Conflicts16gather_conflicts
+__RNvXsf_NtNtCs2NHf1dqD6Ob_4core5slice3cmpNtNtNtCs8ktk9OOIm5l_12clap_builder4util2id2IdNtB5_13SliceContains14slice_containsBI_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecRNtNtNtCs8ktk9OOIm5l_12clap_builder4util2id2IdE8grow_oneBT_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtNtCs8ktk9OOIm5l_12clap_builder6parser7matches11matched_arg10MatchedArgECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecIBC_NtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringEEECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringEECsdSCkLiPtoQj_11weavepy_cli
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser19simple_keyword_stmt
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser14parse_comp_for
+__RNvNtCs9DEk322Hfll_16weavepy_compiler8validate20collect_name_targets
+__RNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB4_9Validator13record_walrus
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCslXHHUCZmUMI_14weavepy_parser3ast13ComprehensionE8grow_oneBQ_
+__RNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB4_9Validator19visit_comprehension
+__RNvNtCs9DEk322Hfll_16weavepy_compiler8validate22collect_walrus_targets
+__RNvNvCs9DEk322Hfll_16weavepy_compiler27collect_comp_scope_walruses5visit
+__RINvXNvMNtCsdq5mISKDh0s_5alloc5sliceSp9to_vec_inNtNtCslXHHUCZmUMI_14weavepy_parser3ast13ComprehensionNtB3_10ConvertVec6to_vecNtNtB8_5alloc6GlobalECs9DEk322Hfll_16weavepy_compiler
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler17compile_class_def
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler19compile_class_value
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler16build_class_body
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCs9DEk322Hfll_16weavepy_compiler10ExcHandlerE8grow_oneBO_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types23install_descriptor_init
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject13set_metaclass
+__RNvCs9DEk322Hfll_16weavepy_compiler25body_reads_super_or_class
+__RNCNvNCNvCs9DEk322Hfll_16weavepy_compiler25body_reads_super_or_class09any_class0B7_
+__RNvCs9DEk322Hfll_16weavepy_compiler19function_free_names
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler18cell_or_free_index
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCslXHHUCZmUMI_14weavepy_parser3ast4ExprEECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast9TypeParamEECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast7KeywordEECs9DEk322Hfll_16weavepy_compiler
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser15parse_class_def
+__RNCNvMs_NtCs9DEk322Hfll_16weavepy_compiler8validateNtB6_9Validator10visit_stmt0B8_
+__RNvNtCs9DEk322Hfll_16weavepy_compiler6mangle17mangle_class_body
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtReNtB6_7Display3fmtCs9DEk322Hfll_16weavepy_compiler
+__RNvMNtCs9DEk322Hfll_16weavepy_compiler6mangleNtB2_7Mangler4stmt
+__RNvMNtCs9DEk322Hfll_16weavepy_compiler6mangleNtB2_7Mangler4name
+__RNvMNtCs9DEk322Hfll_16weavepy_compiler6mangleNtB2_7Mangler9arguments
+__RNvMNtCs9DEk322Hfll_16weavepy_compiler6mangleNtB2_7Mangler4expr
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs9DEk322Hfll_16weavepy_compiler6mangle7ManglerEBF_
+__RNCNvCs9DEk322Hfll_16weavepy_compiler32class_body_defs_claim_class_cell0B3_
+__RNvCs9DEk322Hfll_16weavepy_compiler23method_references_class
+__RNvCs9DEk322Hfll_16weavepy_compiler20stmt_needs_classdict
+__RNvNvCs9DEk322Hfll_16weavepy_compiler23class_body_reads_dunder10stmt_reads
+__RNvXs2_CskHi7NRqR3VY_10weavepy_vmNtB5_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7default
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins16default_builtins
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object19py_hash_bytes_slice
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm6object8HashAlgoE10initializeNCINvB2_11get_or_initNCNvBV_9hash_algo0E0zEBX_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm6object8HashAlgoE10initializeNCINvB1a_11get_or_initNCNvB1I_9hash_algo0E0zE0E0B1K_
+__RNvXsG_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_7DictKeyNtNtCs2NHf1dqD6Ob_4core3cmp9PartialEq2eq
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object7is_same
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object8eq_value
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object19key_needs_interp_eq
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object26instance_has_custom_dunder
+__RINvYNtNtCskHi7NRqR3VY_10weavepy_vm8fasthash13FxBuildHasherNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtB7_6object7DictKeyEB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object11py_str_hash
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object9BuiltinFnE9drop_slowBK_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm6object9BuiltinFnEBF_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object13py_hash_value
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapjuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6removejECskHi7NRqR3VY_10weavepy_vm
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRjECskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons19interactive_printer
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry16mark_default_new
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry8note_key
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry23register_text_signature
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtReNtB6_7Display3fmtCskHi7NRqR3VY_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvMsl_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_13MethodWrapper3new
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types20install_type_dunders
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins20function_get_builtin
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types24install_gen_name_getsets
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject13new_exception
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types8exc_slot
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCskHi7NRqR3VY_10weavepy_vm12shared_value7ThinArcNtNtB1j_13tuple_storage12TupleStorageEuE16get_or_init_slowNvNvNvMs1w_NtB1j_6objectNtB3c_6Object9new_tuple11EMPTY_TUPLE27___rust_std_internal_init_fnEB1j_
+__RNvNvMNtCskHi7NRqR3VY_10weavepy_vm13builtin_typesNtB4_12BuiltinTypes5build22install_field_defaults
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types28install_syntax_error_dunders3set
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types29install_unicode_error_dunders
+__RNvMNtCskHi7NRqR3VY_10weavepy_vm13builtin_typesNtB2_12BuiltinTypes10as_globals
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types29install_numeric_class_methods7install
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types29install_numeric_class_methods14install_method
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface7install
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins19builtin_type_dunder
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapNtNtCsdq5mISKDh0s_5alloc6string6StringINtNtB1j_6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEEuE16get_or_init_slowNvNvNvNtB3y_8builtins19builtin_type_dunder12DUNDER_CACHE27___rust_std_internal_init_fnEB3y_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14unbound_method
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_9slot_callEB4_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtCsdq5mISKDh0s_5alloc6string6StringECskHi7NRqR3VY_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCsdq5mISKDh0s_5alloc6string6StringINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1s_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB28_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface16insert_if_absent
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface7builtin
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface21install_named_methods
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins13lookup_method
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_9str_splitEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_10str_rsplitEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_14str_splitlinesEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_14str_replace_kwEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_17str_expandtabs_kwEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_10str_encodeEB4_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface11unwrap_shim0EBH_
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object8new_list
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCskHi7NRqR3VY_10weavepy_vm12shared_value7ThinArcNtNtBG_13tuple_storage12TupleStorageEEBG_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCskHi7NRqR3VY_10weavepy_vm12shared_value7ThinArcINtBW_12SliceStoragehEEE10initializeNCINvB2_11get_or_initNCINvMs1w_NtBY_6objectNtB2P_6Object9new_bytesINtNtCsdq5mISKDh0s_5alloc3vec3VechEE0E0zEBY_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_15bytes_rsplit_kwEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_19bytes_splitlines_kwEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_16bytes_expandtabsEB4_
+__RINvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB7_6Object13new_bytearrayINtNtCsdq5mISKDh0s_5alloc3vec3VechEEB9_
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface23install_numeric_getsets6getset
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object9PyComplexE9drop_slowBK_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface25register_descriptor_kinds
+__RNvMsl_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_13MethodWrapper4func
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry6lookup
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry20is_slot_wrapper_name
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm14descr_registry9DescrMetaEBF_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface26install_docs_table_surface
+__RNvNtCskHi7NRqR3VY_10weavepy_vm17builtin_docs_data15builtin_doc_for
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry6forget
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons15not_implemented
+__RNvMNtCskHi7NRqR3VY_10weavepy_vm13builtin_typesNtB2_12BuiltinTypes5build
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5types10compute_c3
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry8register
+__RNvYNCNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry10DESCR_META0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceuE9call_onceB8_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry18set_builtin_module
+__RNvMsl_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_13MethodWrapper16wrapper_dict_for
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types20install_type_dunders9mk_getset
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRjECs9DEk322Hfll_16weavepy_compiler
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types22install_module_methods7builtin
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object9new_tuple
+__RINvMNtCskHi7NRqR3VY_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj0_EB5_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEB1c_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtB7_3vec3VecNtNtBL_6object6ObjectEEE9drop_slowBL_
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object8new_dict
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellNtNtBL_6object8DictDataEE9drop_slowBL_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm6object8DictDataEBF_
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object7new_set
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtCs2DfPjnIE6AW_8indexmap3set8IndexSetNtNtBL_6object7DictKeyNtNtBL_8fasthash13FxBuildHasherEEE9drop_slowBL_
+__RINvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB7_6Object9new_bytesINtNtCsdq5mISKDh0s_5alloc3vec3VechEEB9_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCskHi7NRqR3VY_10weavepy_vm12shared_value7ThinArcINtB1J_12SliceStoragehEEE10initializeNCINvB1a_11get_or_initNCINvMs1w_NtB1L_6objectNtB3E_6Object9new_bytesINtNtCsdq5mISKDh0s_5alloc3vec3VechEE0E0zE0E0B1L_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_15bytes_decode_kwEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_12bytes_hex_kwEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_14bytes_split_kwEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_16bytes_replace_kwEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_18bytes_translate_kwEB4_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtB7_3vec3VechEEE9drop_slowBL_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14numeric_dunder
+__RNvXs1_NtNtNtCs2NHf1dqD6Ob_4core3ops8function5implsQNCNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface25register_descriptor_kinds0INtB7_5FnMutTTRNtNtBU_6object7DictKeyRNtB2h_6ObjectEEE8call_mutBU_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types30install_builtin_descriptor_get
+__RNvCskHi7NRqR3VY_10weavepy_vm16builtin_type_doc
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry17mark_surface_only
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject14new_with_flags
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types16make_default_new
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjuEE14reserve_rehashNCINvNtB8_3map11make_hasherjuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4Ez510r8k3d_11weavepy_jit
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface21install_value_richcmp15richcmp_builtin
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface29install_introspection_getsets6getset
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object11new_complex
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtRNtNtCskHi7NRqR3VY_10weavepy_vm12shared_value9SharedStrNtB6_7Display3fmtBA_
+__RNvXsf_NtNtCs2NHf1dqD6Ob_4core5slice3cmpReNtB5_13SliceContains14slice_containsCskHi7NRqR3VY_10weavepy_vm
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneReECskHi7NRqR3VY_10weavepy_vm
+__RNvCskHi7NRqR3VY_10weavepy_vm20builtin_slot_wrapper
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter8run_site
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14run_site_inner
+__RINvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB7_6Object18new_frozenset_fromINtNtCsdq5mISKDh0s_5alloc3vec3VecBG_EEB9_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object12FrozenSetObjE9drop_slowBK_
+__RNvMs6_NtCskHi7NRqR3VY_10weavepy_vm5tier2NtB5_8JitState12get_compiled
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object13MethodWrapperE9drop_slowBK_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellNtNtBL_6object10PyIteratorEE9drop_slowBL_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm6object10PyIteratorEBF_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm13tuple_storage12TupleStorageE9drop_slowBK_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRTjNtNtCsdq5mISKDh0s_5alloc6string6StringEECsbBbqfalDtZG_12weavepy_capi
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8class_of
+__RNvXs_NtNtNtCs2NHf1dqD6Ob_4core5array4iter10iter_innerAINtNtNtBa_3mem12maybe_uninit11MaybeUninitTNtNtCskHi7NRqR3VY_10weavepy_vm6object7DictKeyNtB1z_6ObjectEEj1_NtB4_11PartialDrop12partial_dropB1B_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_12int_to_bytesEB4_
+__RNvMsi_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_12PyMemoryView10from_bytes
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object12PyMemoryViewE9drop_slowBK_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14run_activation
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize18init_stats_enabled
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter9quiet_run
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter10leaf_burst
+__RNvCskHi7NRqR3VY_10weavepy_vm18constant_to_object
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter4step
+__RNvMs_CskHi7NRqR3VY_10weavepy_vmNtB4_5Frame3pop
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter7name_at
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21globals_missing_owner
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6result6ResultNtCskHi7NRqR3VY_10weavepy_vm11StepOutcomeNtNtBZ_5error12RuntimeErrorEEBZ_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os10build_path
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter9do_import
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6import16resolve_relative
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys11audit_event
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os5build
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os7builtin
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os15initial_environ
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os14fsdecode_osstr
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os16stat_result_type
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject8new_user
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject14validate_bases
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject10solid_base
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os10builtin_kw
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE10initializeNCINvB2_11get_or_initNCNvNtNtBX_6stdlib2os22walk_symlinks_sentinel0E0zEBX_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvNtNtB1K_6stdlib2os22walk_symlinks_sentinel0E0zE0E0B1K_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCNvMNtBa_9lazy_lockINtB1a_8LazyLockNtNtCskHi7NRqR3VY_10weavepy_vm8gc_trace7GcStateE5force0E0B1J_
+__RNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB5_7GcState5track
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRyECskHi7NRqR3VY_10weavepy_vm
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys5build
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBG_6string6StringEECskHi7NRqR3VY_10weavepy_vm
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys18version_info_value
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys22disallow_instantiation
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys7builtin
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort12sort4_stableReNvYB19_NtNtBa_3cmp10PartialOrd2ltECskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry15register_module
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB1A_11Interpreter15locals_fast_off0E0zEB1A_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCsl3bTtyQSXML_8lock_api5mutex5MutexNtNtCsfqmARTQr4sq_11parking_lot9raw_mutex8RawMutexINtNtCs2NHf1dqD6Ob_4core6option6OptionNtCskHi7NRqR3VY_10weavepy_vm11InterpreterEEE10initializeNCINvB1a_11get_or_initNCNvNtB3M_13vm_singletons9seed_slot0E0zE0E0B3M_
+__RNvCskHi7NRqR3VY_10weavepy_vm22effective_ctype_locale
+__RNvMs0_NtCskHi7NRqR3VY_10weavepy_vm3gilNtB5_8GilState3new
+__RNvMs0_NtCskHi7NRqR3VY_10weavepy_vm3gilNtB5_8GilState7acquire
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6stdlib12register_all
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache15builtin_factory
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16push_frame_shell
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier29try_enter
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier221jit_enabled_by_config
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCskHi7NRqR3VY_10weavepy_vm5tier28JitStateEuE16get_or_init_slowNvNvB1R_3JIT27___rust_std_internal_init_fnEB1T_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter30run_until_yield_or_return_impl
+__RNvXNtCskHi7NRqR3VY_10weavepy_vm6importNtB2_11ModuleCacheNtNtCs2NHf1dqD6Ob_4core7default7Default7default
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys13venv_cfg_path
+__RNvNtCskHi7NRqR3VY_10weavepy_vm11stdlib_tree11program_exe
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtB7_4path7PathBufEE10initializeNCINvB2_11get_or_initNvNtCskHi7NRqR3VY_10weavepy_vm11stdlib_tree7resolveE0zEB2s_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtBc_4path7PathBufEE10initializeNCINvB1a_11get_or_initNvNtCskHi7NRqR3VY_10weavepy_vm11stdlib_tree7resolveE0zE0E0B3g_
+__RINvMs16_NtCs6KxzipSu1yP_3std4pathNtB7_4Path4joinNtNtCsdq5mISKDh0s_5alloc6string6StringECskHi7NRqR3VY_10weavepy_vm
+__RINvXNtNtCs10OuM599Pl4_10serde_json5value2deNtB5_5ValueNtNtCskautBTia54x_10serde_core2de11Deserialize11deserializeQINtNtB7_2de12DeserializerNtNtB7_4read7StrReadEECskHi7NRqR3VY_10weavepy_vm
+__RINvNvXs9_NtCs10OuM599Pl4_10serde_json2deINtB8_9MapAccesspENtNtCskautBTia54x_10serde_core2de9MapAccess13next_key_seed12has_next_keyNtNtBa_4read7StrReadECskHi7NRqR3VY_10weavepy_vm
+__RNvXs8_NtCs10OuM599Pl4_10serde_json4readNtB5_7StrReadNtB5_4Read9parse_str
+__RNvMs3_NtCs10OuM599Pl4_10serde_json2deINtB5_12DeserializerNtNtB7_4read7StrReadE13parse_integerCskHi7NRqR3VY_10weavepy_vm
+__RNvMs3_NtCs10OuM599Pl4_10serde_json2deINtB5_12DeserializerNtNtB7_4read7StrReadE12parse_numberCskHi7NRqR3VY_10weavepy_vm
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs10OuM599Pl4_10serde_json
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs10OuM599Pl4_10serde_json5value5ValueECskHi7NRqR3VY_10weavepy_vm
+__RINvMNtCs2NHf1dqD6Ob_4core3stre12trim_matchesNvMNtNtB5_4char7methodsc13is_whitespaceECskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8gc_trace13has_finalizer
+__RNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB5_7GcState10bump_count
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys16build_with_state
+__RNvMNtCskHi7NRqR3VY_10weavepy_vm13tuple_storageNtB2_12TupleStorage8from_vec
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os19struct_seq_instance
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCskHi7NRqR3VY_10weavepy_vm9hot_gates9env_flags8no_quiet0E0zE0E0B2o_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB2o_11Interpreter15locals_fast_off0E0zE0E0B2o_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCsl3bTtyQSXML_8lock_api5mutex5MutexNtNtCsfqmARTQr4sq_11parking_lot9raw_mutex8RawMutexINtNtCs2NHf1dqD6Ob_4core6option6OptionNtCskHi7NRqR3VY_10weavepy_vm11InterpreterEEE10initializeNCINvB2_11get_or_initNCNvNtB2Z_13vm_singletons9seed_slot0E0zEB2Z_
+__RNvMs1l_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6PyFile10set_errors
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCsdq5mISKDh0s_5alloc4sync3ArcNtNtCskHi7NRqR3VY_10weavepy_vm3gil8GilStateEE10initializeNCINvB1a_11get_or_initNCNvB2g_14global_gil_ref0E0zE0E0B2i_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11import_path
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter8load_one
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache21is_third_party_facade
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache28find_source_shadowing_stdlib
+__RINvXsE_NtCs6KxzipSu1yP_3std4pathNtB6_7PathBufINtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits7collect12FromIteratorReE9from_iterINtNtNtBR_3str4iter5SplitcEECskHi7NRqR3VY_10weavepy_vm
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache11search_dirs
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCsdq5mISKDh0s_5alloc3vec9into_iter8IntoIterNtNtCs6KxzipSu1yP_3std4path7PathBufEECskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm11stdlib_tree11module_path
+__RINvMs16_NtCs6KxzipSu1yP_3std4pathNtB7_4Path4joinNtB7_7PathBufECskHi7NRqR3VY_10weavepy_vm
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_mod15load_from_bytes
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader4take
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader8note_ref
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE8grow_oneB1s_
+__RNvMNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13codecs_engineNtB2_6DecCtx3new
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object19str_from_codepoints
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13codecs_engine6DecCtxEBH_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapNtNtCskHi7NRqR3VY_10weavepy_vm12shared_value9SharedStruNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBR_
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object13as_bytes_view
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_mod9string_of
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_mod18tuple_to_constants
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs9DEk322Hfll_16weavepy_compiler12cpython_code10DecodedRawE8grow_oneBQ_
+__RNvNtCs9DEk322Hfll_16weavepy_compiler12cpython_code19decode_instructions
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjjEE14reserve_rehashNCINvNtB8_3map11make_hasherjjNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs9DEk322Hfll_16weavepy_compiler
+__RNvNtCs9DEk322Hfll_16weavepy_compiler12cpython_code15raw_first_instr
+__RNvMsi_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_12PyMemoryView7release
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm6object16MemoryViewBufferEBF_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_15memoryview_castEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_14memoryview_hexEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_18memoryview_tobytesEB4_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object10PyPropertyE9drop_slowBK_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_16property_init_kwEB4_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object5RangeE9drop_slowBK_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object7PySliceE9drop_slowBK_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_17staticmethod_callEB4_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB2_13str_format_kwEB4_
+__RINvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB7_6Object8from_strRNtNtCsdq5mISKDh0s_5alloc6string6StringEB9_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm11stdlib_tree7resolve
+__RNvNtCskHi7NRqR3VY_10weavepy_vm11stdlib_tree13landmark_walk
+__RNvMsi_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8BTreeMapNtNtBb_6string6StringNtNtCs10OuM599Pl4_10serde_json5value5ValueE6insertCskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs10OuM599Pl4_10serde_json5value5ValueE8grow_oneBQ_
+__RNvMsz_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8IntoIterNtNtBb_6string6StringNtNtCs10OuM599Pl4_10serde_json5value5ValueE10dying_nextCskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE8grow_oneBQ_
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache6insert
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8gc_trace5track
+__RNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB5_7GcState3new
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once9call_onceNCNvXs2_CskHi7NRqR3VY_10weavepy_vmNtB16_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_0E0B16_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEBX_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B1K_
+__RINvMNtCskHi7NRqR3VY_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj3_EB5_
+__RINvMs1l_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB7_6PyFile3newReBW_EB9_
+__RNvMs1k_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6IoKind9from_mode
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib14sys_monitoring5build
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5trace20monitoring_sentinels
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5trace20monitoring_sentinels0B5_
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable7ipnsortReNvYBT_NtNtB8_3cmp10PartialOrd2ltECskHi7NRqR3VY_10weavepy_vm
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable9quicksort9quicksortReNvYB17_NtNtBa_3cmp10PartialOrd2ltECskHi7NRqR3VY_10weavepy_vm
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared5pivot11median3_recReNvYB14_NtNtBa_3cmp10PartialOrd2ltECskHi7NRqR3VY_10weavepy_vm
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCskHi7NRqR3VY_10weavepy_vm9hot_gates9env_flags8no_quiet0E0zEB1A_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCskHi7NRqR3VY_10weavepy_vm9hot_gates9env_flags8no_burst0E0zEB1A_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCskHi7NRqR3VY_10weavepy_vm9hot_gates9env_flags8no_burst0E0zE0E0B2o_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvNtNtCskHi7NRqR3VY_10weavepy_vm9hot_gates9env_flags10dbg_sample0E0zEB1A_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvNtNtCskHi7NRqR3VY_10weavepy_vm9hot_gates9env_flags10dbg_sample0E0zE0E0B2o_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons24publish_interpreter_seed
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15fork_for_thread
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17apply_run_options
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCsdq5mISKDh0s_5alloc4sync3ArcNtNtCskHi7NRqR3VY_10weavepy_vm3gil8GilStateEE10initializeNCINvB2_11get_or_initNCNvB1t_14global_gil_ref0E0zEB1v_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCskHi7NRqR3VY_10weavepy_vm3gil8GilGuardE8grow_oneBQ_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecONtCskHi7NRqR3VY_10weavepy_vm11InterpreterE8grow_oneBP_
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache3get
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache19initializing_holder
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache17try_begin_loading
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapNtNtCsdq5mISKDh0s_5alloc6string6StringyNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCskHi7NRqR3VY_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCsdq5mISKDh0s_5alloc6string6StringyEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_yNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECskHi7NRqR3VY_10weavepy_vm
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache13live_sys_path
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6import19is_test_frozen_name
+__RNvNtCskHi7NRqR3VY_10weavepy_vm17frozen_code_cache3get
+__RNvNtCskHi7NRqR3VY_10weavepy_vm17frozen_code_cache8get_disk
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtB7_4path7PathBufEE10initializeNCINvB2_11get_or_initNCNvNtCskHi7NRqR3VY_10weavepy_vm17frozen_code_cache8disk_dir0E0zEB2u_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtBc_4path7PathBufEE10initializeNCINvB1a_11get_or_initNCNvNtCskHi7NRqR3VY_10weavepy_vm17frozen_code_cache8disk_dir0E0zE0E0B3i_
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader10read_value
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader9read_byte
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader9read_code
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader8read_int
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10codecs_mod16decode_bytes_obj
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtCskHi7NRqR3VY_10weavepy_vm12shared_value9SharedStrEB1J_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCskHi7NRqR3VY_10weavepy_vm12shared_value9SharedStruEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBU_
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_modNtB5_13MarshalReader8fill_ref
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_mod8bytes_of
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_mod16tuple_of_strings
+__RNvCskHi7NRqR3VY_10weavepy_vm18object_to_constant
+__RNvNtCs9DEk322Hfll_16weavepy_compiler12cpython_code11decode_full
+__RNvNtCs9DEk322Hfll_16weavepy_compiler12cpython_code11decode_raws
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_mod13MarshalReaderEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEEB1y_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm7pycache16own_decoded_code
+__RNvNtCskHi7NRqR3VY_10weavepy_vm7pycache17rewrite_filenames
+__RNvNtCskHi7NRqR3VY_10weavepy_vm17frozen_code_cache6insert
+__RNvMs_NtCsdq5mISKDh0s_5alloc3vecINtB4_3VechE16into_boxed_sliceCskHi7NRqR3VY_10weavepy_vm
+__RNvXsX_Cs9DEk322Hfll_16weavepy_compilerNtB5_10CodeObjectNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19run_frozen_compiled
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20build_module_globals
+__RNvNtCskHi7NRqR3VY_10weavepy_vm7pycache14cache_path_for
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache18begin_initializing
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter10make_frame
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter23pooled_locals_from_args
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE6resizeBJ_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15recycle_scratch
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object11empty_cells
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter9run_frame
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter25run_until_yield_or_return
+__RNvCs9IcUwEW2Om8_7stacker15remaining_stack
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell4CellINtNtB1j_6option6OptionjEEzE16get_or_init_slowNvNvCs9IcUwEW2Om8_7stacker11STACK_LIMIT27___rust_std_internal_init_fnEB2A_
+__RNvCs9DEk322Hfll_16weavepy_compiler20comp_clause_is_async
+__RNvCs9DEk322Hfll_16weavepy_compiler19expr_contains_await
+__RNvCs9DEk322Hfll_16weavepy_compiler24expr_contains_async_comp
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler21compile_comprehension
+__RNvCs9DEk322Hfll_16weavepy_compiler22check_comp_walrus_nest
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtCs9DEk322Hfll_16weavepy_compiler15CompWalrusScopeE8grow_oneBO_
+__RNvNvCs9DEk322Hfll_16weavepy_compiler25reject_walrus_in_iterable4scan
+__RNvNvCs9DEk322Hfll_16weavepy_compiler22check_comp_walrus_nest5visit
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs9DEk322Hfll_16weavepy_compiler9flowgraph11HandlerInfoE8grow_oneBQ_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCs9DEk322Hfll_16weavepy_compiler15CompWalrusScopeEEB1a_
+__RNvCs9DEk322Hfll_16weavepy_compiler17compile_comp_body
+__RNvCs9DEk322Hfll_16weavepy_compiler19stamp_comp_elt_span
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler21note_identifier_const
+__RNvMs0_NtCs9DEk322Hfll_16weavepy_compiler9flowgraphNtB5_3Cfg15make_load_const
+__RNvNtCs9DEk322Hfll_16weavepy_compiler7ast_opt18eval_unaryop_const
+__RNvNtCs9DEk322Hfll_16weavepy_compiler9flowgraph15to_ast_constant
+__RINvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_stmt_inners_0EB6_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCslXHHUCZmUMI_14weavepy_parser3ast7KeywordE8grow_oneBQ_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm3gil15maybe_yield_gil
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10ext_loader24current_extension_loader
+__RNvNtCsbBbqfalDtZG_12weavepy_capi6loader17vm_load_extension
+__RNvNtCsbBbqfalDtZG_12weavepy_capi5embed14inittab_lookup
+__RINvMNtNtNtCs6KxzipSu1yP_3std3sys4sync8once_boxINtB3_7OnceBoxNtNtNtNtNtB7_3pal4unix4sync5mutex5MutexE10initializeNCNvMNtNtB5_5mutex7pthreadNtB1S_5Mutex3get0ECsbBbqfalDtZG_12weavepy_capi
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10ext_loader18take_explicit_path
+__RNvNtCsbBbqfalDtZG_12weavepy_capi6loader22find_extension_on_path
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VechEECsbBbqfalDtZG_12weavepy_capi
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtNtB9_3ffi6os_str8OsStringECskHi7NRqR3VY_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs6KxzipSu1yP_3std3ffi6os_str8OsStringuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtNtNtBW_4hash6random11RandomStateE0ECskHi7NRqR3VY_10weavepy_vm
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler28compile_deferred_annotations
+__RNvNvCs9DEk322Hfll_16weavepy_compiler24collect_self_attr_stores6target
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler13closure_index
+__RNvCs9DEk322Hfll_16weavepy_compiler24collect_self_attr_stores
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10os_process8register
+__RINvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB7_6Object12new_set_fromINtNtCsdq5mISKDh0s_5alloc3vec3VecBG_EEB9_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object13identity_hash
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecyE8grow_oneCslSKxsPn8Dnr_10num_bigint
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCslSKxsPn8Dnr_10num_bigint
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache19builtin_static_name
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter12bind_os_path
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib9errno_mod5build
+__RNvNtNtCslSKxsPn8Dnr_10num_bigint7biguint5shift12biguint_shl2
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNvMs3_CskHi7NRqR3VY_10weavepy_vmNtBJ_11Interpreter8load_one12LoadingGuardEBJ_
+__RNvXNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB8_11Interpreter8load_oneNtB2_12LoadingGuardNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24lookup_global_or_builtin
+__RNvMsh_NtCslSKxsPn8Dnr_10num_bigint7biguintNtB5_7BigUint10normalized
+__RNvXNtNtCslSKxsPn8Dnr_10num_bigint7biguint8additionNtB4_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3AddRBN_E3add
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object15int_from_bigint
+__RINvMs3_CskHi7NRqR3VY_10weavepy_vmNtB6_11Interpreter17alloc_tuple_arrayKj2_EB6_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15leaf_burst_slow
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11alloc_tuple
+__RINvMs3_CskHi7NRqR3VY_10weavepy_vmNtB6_11Interpreter17alloc_tuple_arrayKj3_EB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtBG_5boxed3BoxNtCskHi7NRqR3VY_10weavepy_vm9InlineActEEEB1s_
+__RINvMNtCskHi7NRqR3VY_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj2_EB5_
+__RINvMs3_CskHi7NRqR3VY_10weavepy_vmNtB6_11Interpreter17alloc_tuple_arrayKj1_EB6_
+__RINvMNtCskHi7NRqR3VY_10weavepy_vm13tuple_storageNtB3_12TupleStorage10from_arrayKj1_EB5_
+__RNvMsf_NtNtCs2NHf1dqD6Ob_4core3str4iterINtB5_13SplitInternalcE9next_backCsbBbqfalDtZG_12weavepy_capi
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtB4_2io5error5ErrorECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionTNtNtCs6KxzipSu1yP_3std4path7PathBufINtNtCsdq5mISKDh0s_5alloc2rc2RcINtNtNtNtB12_11collections4hash3set7HashSetNtNtNtB12_3ffi6os_str8OsStringEEEEECsbBbqfalDtZG_12weavepy_capi
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapNtNtBe_4path7PathBufTNtNtCskHi7NRqR3VY_10weavepy_vm6import8DirStampINtNtCsdq5mISKDh0s_5alloc2rc2RcINtNtB1U_3set7HashSetNtNtNtBe_3ffi6os_str8OsStringEEEEEuE16get_or_init_slowNvNvB2S_12DIR_LISTINGS27___rust_std_internal_init_fnEB2U_
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtB9_4path7PathBufECskHi7NRqR3VY_10weavepy_vm
+__RINvXs1h_NtCs6KxzipSu1yP_3std4pathNtB7_4PathNtNtCs2NHf1dqD6Ob_4core4hash4Hash4hashNtNtNtB9_4hash6random13DefaultHasherECskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6import11dir_listing
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtRReNtB6_7Display3fmtCsbBbqfalDtZG_12weavepy_capi
+__RNvXs3_NtNtCs2NHf1dqD6Ob_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCsbBbqfalDtZG_12weavepy_capi
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache11find_source
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache22find_namespace_package
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs6KxzipSu1yP_3std4path7PathBufEECskHi7NRqR3VY_10weavepy_vm
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20try_meta_path_import
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache16end_initializing
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter4call
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier222try_call_native_direct
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter23specialized_load_global
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize30attempt_specialize_load_global
+__RINvMNtNtCs2NHf1dqD6Ob_4core4cell4onceINtB3_8OnceCellNtCskHi7NRqR3VY_10weavepy_vm7LeafFnsE8try_initNCINvB2_11get_or_initNCNvMs3_BR_NtBR_11Interpreter8leaf_fns0E0zEBR_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjNtCskHi7NRqR3VY_10weavepy_vm8LeafKindNtNtBQ_8fasthash13FxBuildHasherE6insertBQ_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6result6ResultNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectNtNtB11_5error12RuntimeErrorEEB11_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier221try_call_native_bound
+__RNvXs_NtCskHi7NRqR3VY_10weavepy_vm10specializeNtNtB6_6object8DictDataNtB4_11DictDataExt16index_of_key_str
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14str_rpartitionINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14str_rpartition
+__RINvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types14make_exceptionNtNtCsdq5mISKDh0s_5alloc6string6StringEB4_
+__RNvMNtCskHi7NRqR3VY_10weavepy_vm13builtin_typesNtB2_12BuiltinTypes7by_name
+__RINvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types25make_exception_with_classNtNtCsdq5mISKDh0s_5alloc6string6StringEB4_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types19is_subclass_by_name
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object6to_str
+__RNvMsj_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10PyInstance8slot_set
+__RNvMsj_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10PyInstance17ensure_gc_tracked
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5types8SlotDataEBF_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtCskHi7NRqR3VY_10weavepy_vm6object7DictKeyNtBP_6ObjectEE8grow_oneBR_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5error18set_exception_attr
+__RNvMsj_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10PyInstance8slot_get
+__RNvMsi_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_11SlotStorage3get
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter23attach_implicit_context
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell4CellTymEEE4withNCNvNtCskHi7NRqR3VY_10weavepy_vm5trace28monitoring_union_mask_cached0mEB1J_
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellNtNtBZ_5trace15MonitoringToolsEE4withNCNvB1E_21monitoring_union_mask0mEBZ_
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellNtNtB1j_5trace15MonitoringToolsEuE16get_or_init_slowNvNvB1Y_16MONITORING_TOOLS27___rust_std_internal_init_fnEB1j_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16handle_exception
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCskHi7NRqR3VY_10weavepy_vm5error14TracebackEntryE8grow_oneBQ_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object20materialize_stack_at
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5error11PyExceptionEBF_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm5error14TracebackEntryEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5error14TracebackEntryEBF_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19ensure_top_py_frame
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object21materialize_stack_top
+__RNvXsf_NtCskHi7NRqR3VY_10weavepy_vm5errorNtB5_11PyExceptionNtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15pop_frame_shell
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19recycle_frame_shell
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20recycle_frame_allocs
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE8grow_oneB16_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCskHi7NRqR3VY_10weavepy_vm5FrameEBD_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm5error11PyExceptionEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCskHi7NRqR3VY_10weavepy_vm5tier216NativeActivationEEEB1A_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17call_python_owned
+__RNvMs12_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_10PyFunction4code
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16leaf_core_helper
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14leaf_core_attr
+__RNvMs9_NtCs9DEk322Hfll_16weavepy_compiler8bytecodeNtB5_10CacheTable8allocate
+__RNvMs5_NtCs9DEk322Hfll_16weavepy_compiler8bytecodeNtB5_9CacheSlot3new
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14leaf_core_call
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjNtCskHi7NRqR3VY_10weavepy_vm8LeafKindEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtBT_8fasthash13FxBuildHasherE0EBT_
+__RNvCskHi7NRqR3VY_10weavepy_vm17object_dunder_new
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18load_method_ic_hit
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21specialized_load_attr
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter9load_attr
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15load_attr_inner
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter10maybe_bind
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize28attempt_specialize_load_attr
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15try_inline_call
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13try_lean_call
+__RNvMs_NtCsdq5mISKDh0s_5alloc3vecINtB4_3VecNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE6removeBI_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13dispatch_call
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14pooled_scratch
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize23attempt_specialize_call
+__RNvCskHi7NRqR3VY_10weavepy_vm20builtin_needs_interp
+__RNvCskHi7NRqR3VY_10weavepy_vm21code_pure_leaf_decide
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18call_from_bytecode
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter9core_call
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15call_c_profiled
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10str_result
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object11BoundMethodE9drop_slowBK_
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object9make_iter
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE6borrowB7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE10borrow_mutB7_
+__RNvMs1q_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_10PyIterator18next_value_checked
+__RNvMs1q_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_10PyIterator10next_value
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17try_lean_property
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14leaf_type_call
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtBG_6string6StringNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEEB1y_
+__RNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB5_7GcState18maybe_auto_collect
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14leaf_call_kind
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object3len
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object12str_char_len
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17try_lean_for_iter
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20specialized_for_iter
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize27attempt_specialize_for_iter
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8gc_trace16strong_count_for
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject14is_subclass_of
+__RNvNtCskHi7NRqR3VY_10weavepy_vm16weakref_registry9count_for
+__RNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB5_7GcState22rebuild_tracked_filter
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter23activate_thread_handles
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons24current_worker_thread_id
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCsl3bTtyQSXML_8lock_api5mutex5MutexNtNtCsfqmARTQr4sq_11parking_lot9raw_mutex8RawMutexINtNtNtNtB7_11collections4hash3map7HashMapyyEEE10initializeNCINvB2_11get_or_initNCNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons10worker_map0E0zEB3J_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib16faulthandler_mod17note_thread_start
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib16faulthandler_mod16RegisteredThreadE8grow_oneBS_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons13ThreadHandlesE8grow_oneBQ_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object23collect_small_int_range
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_lenINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMst_NtCskHi7NRqR3VY_10weavepy_vm12shared_valueNtB5_9SharedStr6concat
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17type_call_default
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11instantiate
+__RNvCskHi7NRqR3VY_10weavepy_vm20object_needs_vm_iter
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCsl3bTtyQSXML_8lock_api5mutex5MutexNtNtCsfqmARTQr4sq_11parking_lot9raw_mutex8RawMutexINtNtNtNtBc_11collections4hash3map7HashMapyyEEE10initializeNCINvB1a_11get_or_initNCNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons10worker_map0E0zE0E0B4x_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16collect_iterable
+__RNvXs0_NtNtCsdq5mISKDh0s_5alloc3vec11spec_extendINtB7_3VecNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEINtB5_10SpecExtendBU_INtNtB7_9into_iter8IntoIterBU_EE11spec_extendBY_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14leaf_fused_len
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17leaf_builtin_call
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_arity
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject17metaclass_or_type
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins6b_typeINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMs11_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_10PyFunction14lean_cells_ref
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter10lean_frame
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14run_frame_lean
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19run_frame_lean_impl
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15flush_lean_cold
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15push_lean_shell
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17retire_lean_frame
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24specialized_store_subscr
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15ensure_dict_key
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15ensure_hashable
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11dict_insert
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons23current_interpreter_ptr
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object15dict_watch_bump
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object19dict_mutation_event
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6result6ResultINtNtB4_6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectENtNtB1n_5error12RuntimeErrorEEB1n_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons13ThreadHandlesEBF_
+__RNvXs0_NtCskHi7NRqR3VY_10weavepy_vm13vm_singletonsNtB5_16InterpreterGuardNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20emit_escape_warnings
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13run_module_as
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter22run_pending_finalizers
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm3gil8GilGuardEECsbBbqfalDtZG_12weavepy_capi
+__RNvXsu_CsbuKYNc0jrX_4itoayNtB5_8Unsigned3fmt
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCs6KxzipSu1yP_3std4path7PathBufTNtNtCskHi7NRqR3VY_10weavepy_vm6import8DirStampINtNtCsdq5mISKDh0s_5alloc2rc2RcINtNtNtNtBU_11collections4hash3set7HashSetNtNtNtBU_3ffi6os_str8OsStringEEEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1p_NtNtNtBU_4hash6random11RandomStateE0EB1u_
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtReNtB6_7Display3fmtCsbBbqfalDtZG_12weavepy_capi
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCskHi7NRqR3VY_10weavepy_vm5error11PyExceptionE8grow_oneBQ_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17exception_matches
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types20instance_is_subclass
+__RNvYNCNCNvXs2_CskHi7NRqR3VY_10weavepy_vmNtBc_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_00INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
+__RNvYNCNCNvXs2_CskHi7NRqR3VY_10weavepy_vmNtBc_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_0s_0INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
+__RNvYNCNCNvXs2_CskHi7NRqR3VY_10weavepy_vmNtBc_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_0s1_0INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
+__RNvCskHi7NRqR3VY_10weavepy_vm17py_frame_traverse
+__RNvXsk_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10PyInstanceNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellNtNtBL_6object6ObjectEE9drop_slowBL_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCNvMNtBa_9lazy_lockINtB1a_8LazyLockNtNtCskHi7NRqR3VY_10weavepy_vm16weakref_registry15WeakRefRegistryE5force0E0B1J_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16pop_jump_if_step
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object9is_truthy
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter12pop_top_step
+__RNvMsi_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_11SlotStorage4iter
+__RNvYNCNCNvXs2_CskHi7NRqR3VY_10weavepy_vmNtBc_11InterpreterNtNtCs2NHf1dqD6Ob_4core7default7Default7defaults0_0s0_0INtNtNtBY_3ops8function6FnOnceTRNtNtBc_6object6ObjectEE9call_onceBc_
+__RNvCskHi7NRqR3VY_10weavepy_vm21py_traceback_traverse
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm5types10PyInstanceE9drop_slowBK_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object11PyTracebackE9drop_slowBK_
+__RNvXse_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_11PyTracebackNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMs3_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_7PyFrame14refresh_locals
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object7PyFrameE9drop_slowBK_
+__RNvXs2_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_7PyFrameNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object10FrameShellE9drop_slowBK_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_splitE9drop_slowBK_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14str_replace_kwE9drop_slowBK_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object16is_function_slot
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize29attempt_specialize_store_attr
+__RNvMs3_NtCskHi7NRqR3VY_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry3new
+__RNvCskHi7NRqR3VY_10weavepy_vm16metaclass_method
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_dir
+__RNvMsi_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8BTreeMapNtNtBb_6string6StringNtNtB7_7set_val9SetValZSTE6insertCskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14str_startswith
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11list_appendINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier213try_enter_osr
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13contains_full
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_setINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvCskHi7NRqR3VY_10weavepy_vm5tsdbg0E0zEB1w_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13sort_with_key
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8dict_get
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16module_shadowing
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11import_from
+__RINvXNtNtCsdq5mISKDh0s_5alloc3vec14spec_from_elemNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectNtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEBP_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter9make_iter
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11b_frozensetINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject17lookup_with_owner
+__RNvMs3_NtCskHi7NRqR3VY_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry14collect_strong
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter25specialized_binary_subscr
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10b_reversed
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object9as_bigint
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15weave_frame_mod5build
+__RNvYFG_RL0_SNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEINtNtCs2NHf1dqD6Ob_4core6result6ResultBa_NtNtBe_5error12RuntimeErrorEINtNtNtBY_3ops8function2FnTRB9_EE4callBe_
+__RNvMs1d_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_11PyGenerator11is_finished
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16gen_method_close
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface24install_type_dict_extras0B5_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14sync_py_locals
+__RNvNtCskHi7NRqR3VY_10weavepy_vm16weakref_registry15queue_callbacks
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types16make_default_new0B5_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types10object_new
+__RINvNtNtCs2NHf1dqD6Ob_4core4iter8adapters11try_processINtNtB2_3map3MapQDNtNtNtB4_6traits8iterator8Iteratorp4ItemRNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEL_NCNCNvMs3_B1R_NtB1R_11Interpreter11import_star00ENtNtCsdq5mISKDh0s_5alloc6string6StringINtNtB6_6result6ResultNtNtB6_7convert10InfallibleNtNtB1R_5error12RuntimeErrorENCINvXso_B40_IB3Y_INtNtB3p_3vec3VecB3l_EB4K_EINtNtB1c_7collect12FromIteratorIB3Y_B3l_B4K_EE9from_iterBQ_E0B5v_EB1R_
+__RNvXs2J_NtNtCs2NHf1dqD6Ob_4core5slice4iterINtB6_4IterNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectENtNtNtNtBa_4iter6traits8iterator8Iterator4nextBT_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter10store_attr
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_dirINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins19str_apply_start_end
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15leaf_store_attr
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8dict_getINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real1b
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject10subclasses
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins17build_super_proxy
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17user_getattribute
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter29store_attr_instance_with_name
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real13construct_ref
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_class
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9dict_keysINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11dict_valuesINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvCskHi7NRqR3VY_10weavepy_vm6i64_op
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecyE6resizeCslSKxsPn8Dnr_10num_bigint
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins7b_range
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys12sys_getframe
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20release_finished_gen
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object13AsyncGenAwaitE9drop_slowBK_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15store_fast_step
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19leaf_load_type_attr
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins7set_addINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_classs0_0EBJ_
+__RNvMsn_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_10PyProperty4fget
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19dispatch_compare_op
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter10cmp_method
+__RNvCskHi7NRqR3VY_10weavepy_vm8set_view
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter10obj_truthy
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19generator_send_lean
+__RNvMs5_Cs9DEk322Hfll_16weavepy_compilerNtB5_7JitHint17every_loop_yields
+__RNvMs3_NtCskHi7NRqR3VY_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry5count
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14lean_call_fill
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13inline_finish
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real13ref_type_call
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18inline_gen_deliver
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter22load_method_ic_resolve
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11delete_attr
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter22specialized_store_attr
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15do_getattr_call
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11dict_lookup
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object12module_class
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter23module_spec_file_origin
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18is_attribute_error
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17ensure_os_environ
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real5build
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject17bump_attr_version
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10dict_items
+__RNvMsI_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_9DictWatch3new
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24alloc_plain_instance_obj
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize22type_has_attr_override
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject18setattr_is_default
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapyuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCskHi7NRqR3VY_10weavepy_vm
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject21validate_python_bases
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object10PyFunctionE9drop_slowBK_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18deopt_call_generic
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter22specialized_compare_op
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10b_reversedINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCslSKxsPn8Dnr_10num_bigint6bigint6BigIntE9drop_slowCskHi7NRqR3VY_10weavepy_vm
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15weave_frame_mod9frame_arg
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18maybe_donate_tuple
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectB19_EEEB1d_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object11PyGeneratorE9drop_slowBK_
+__RNvCskHi7NRqR3VY_10weavepy_vm15make_gen_method
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15agen_init_hooks
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins20apply_set_inplace_op
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18do_issubclass_call
+__RINvMs3_NtCs2DfPjnIE6AW_8indexmap3mapINtB6_8IndexMapNtNtCskHi7NRqR3VY_10weavepy_vm6object7DictKeyuNtNtBS_8fasthash13FxBuildHasherE12get_index_ofBO_ECsbBbqfalDtZG_12weavepy_capi
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry24is_native_descr_accessor
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15reentrant_py_eq
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCskHi7NRqR3VY_10weavepy_vm10QuietEntryEBD_
+__RINvNtCskHi7NRqR3VY_10weavepy_vm5error10type_errorNtNtCsdq5mISKDh0s_5alloc6string6StringEB4_
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object15type_name_owned
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins19builtin_classmethod
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface24install_type_dict_extras12type_prepare
+__RNvCskHi7NRqR3VY_10weavepy_vm20mangle_class_private
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24finalize_class_namespace
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBe_11collections4hash3map7HashMapjNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEuE16get_or_init_slowNvNvNtB2A_8builtins15DOCSTRING_CACHE27___rust_std_internal_init_fnEB2A_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15do_hasattr_call
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter23run_py_exact_with_cells
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_set
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24run_py_exact_nofree_with
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real24install_proxy_forwarding
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16try_lean_call_kw
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBV_6string6StringEE10borrow_mutB7_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter22instantiate_plain_lean
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13instance_plan
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real19find_cached_wrapper
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins7b_rangeINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMss_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_5Range12from_bigints
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15weave_frame_mod5check
+__RNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB5_7GcState14note_finalized
+__RNvXs1f_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_11PyGeneratorNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvCskHi7NRqR3VY_10weavepy_vm15make_agen_await
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15leaf_class_attr
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm16weakref_registry11WeakRefSlotE9drop_slowBK_
+__RNvCskHi7NRqR3VY_10weavepy_vm15is_pep604_union
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8set_copy
+__RINvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types25make_exception_with_classReEB4_
+__RNvMs1_NtCskHi7NRqR3VY_10weavepy_vm5errorNtB5_11PyException9type_name
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real16wrapper_referent
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins29class_matches_classinfo_named
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins19type_subclass_match
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtCskHi7NRqR3VY_10weavepy_vm17InlinePlaceholderE10initializeNCINvB1a_11get_or_initNCNvB1I_18inline_placeholder0E0zE0E0B1I_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtB7_5boxed3BoxNtCskHi7NRqR3VY_10weavepy_vm9InlineActEE8grow_oneB16_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19binary_subscr_basic
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20del_type_attr_direct
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject22qualified_display_name
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys25sys_getfilesystemencoding
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter9iter_next
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvBG_10str_encodeE0EBI_
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject13lookup_cached
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14descriptor_get
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11build_class
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter22call_metaclass_prepare
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20invoke_init_subclass
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter26check_classcell_propagated
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTyuEE14reserve_rehashNCINvNtB8_3map11make_hasheryuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECskHi7NRqR3VY_10weavepy_vm
+__RNvMs3_NtCskHi7NRqR3VY_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry12notify_clear
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types19native_seed_for_new
+__RNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB7_11Interpreter9do_imports0_0B7_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectbEEEB1d_
+__RNvMs12_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_10PyFunction4slot
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14code_docstring
+__RNvMsz_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8IntoIterNtNtBb_6string6StringNtNtB7_7set_val9SetValZSTE10dying_nextCskHi7NRqR3VY_10weavepy_vm
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14str_startswithINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins23str_match_prefix_suffix
+__RNvCskHi7NRqR3VY_10weavepy_vm12attr_name_of
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11list_append
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object8contains
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins18ensure_set_element
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14binary_op_step
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21specialized_binary_op
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18dispatch_binary_op
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvCskHi7NRqR3VY_10weavepy_vm5tsdbg0E0zE0E0B2k_
+__RNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB7_11Interpreter18dispatch_binary_ops3_0B7_
+__RNvCskHi7NRqR3VY_10weavepy_vm19dict_view_set_elems
+__RNvCskHi7NRqR3VY_10weavepy_vm9binary_op
+__RNvXs5_NtCsdq5mISKDh0s_5alloc5sliceSINtCs2DfPjnIE6AW_8indexmap6BucketNtNtCskHi7NRqR3VY_10weavepy_vm6object7DictKeyuEINtB5_16SpecCloneIntoVecBy_NtNtB7_5alloc6GlobalE10clone_intoB19_
+__RNvCskHi7NRqR3VY_10weavepy_vm10union_sets
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14do_sorted_call
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object3cmp
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15dict_setdefaultINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15dict_setdefault
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectbEE8grow_oneBR_
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache15is_initializing
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types8exc_attr
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtRNtNtCsdq5mISKDh0s_5alloc6string6StringNtB6_7Display3fmtCskHi7NRqR3VY_10weavepy_vm
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19leaf_load_attr_recv
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15compare_op_step
+__RNvCskHi7NRqR3VY_10weavepy_vm11compare_str
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins21construct_classmethod
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20set_type_attr_direct
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter26metaclass_descriptor_store
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18try_inline_call_kw
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18lean_call_kw_shape
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11build_super
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter26load_attr_instance_default
+__RNvCskHi7NRqR3VY_10weavepy_vm21callable_function_str
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter27dynamic_type_call_with_meta
+__RNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB7_11Interpreter24finalize_class_namespaces_0B7_
+__RNvMsf_NtNtCs2NHf1dqD6Ob_4core3str4iterINtB5_13SplitInternalcE9next_backCskHi7NRqR3VY_10weavepy_vm
+__RINvMNtCs2NHf1dqD6Ob_4core3stre18trim_start_matchescECskHi7NRqR3VY_10weavepy_vm
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10dict_itemsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvCskHi7NRqR3VY_10weavepy_vm16is_generic_alias
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object10PyDictViewE9drop_slowBK_
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object9type_name
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCskHi7NRqR3VY_10weavepy_vm4sync4CellyEE9drop_slowBL_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11b_frozenset
+__RNvCskHi7NRqR3VY_10weavepy_vm11bind_method
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter8core_new
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry14is_default_new
+__RNvMsj_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10PyInstance12new_deferred
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter34generic_setattr_instance_with_name
+__RNvMsN_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_8DictData20track_deferred_owner
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real16supports_weakref
+__RNvMs3_NtCskHi7NRqR3VY_10weavepy_vm16weakref_registryNtB5_15WeakRefRegistry8register
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm5types10PyInstanceE9downgradeBK_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueTNtNtCsdq5mISKDh0s_5alloc6string6StringNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEB1i_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15subscr_get_step
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface21class_getitem_builtin
+__RNvCskHi7NRqR3VY_10weavepy_vm18make_generic_alias
+__RNvCskHi7NRqR3VY_10weavepy_vm23collect_type_parameters
+__RNvCskHi7NRqR3VY_10weavepy_vm15is_typevar_like
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11b_bytearrayINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14b_bytearray_kw
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15bytes_construct
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9dict_keys
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11dict_values
+__RNvCskHi7NRqR3VY_10weavepy_vm9bignum_op
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object6as_i64
+__RNvXs2_NtNtCslSKxsPn8Dnr_10num_bigint6bigint7convertNtB7_6BigIntINtNtCs2NHf1dqD6Ob_4core7convert4FromxE4from
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins19coerce_index_bigint
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCslSKxsPn8Dnr_10num_bigint6bigint6BigIntEECskHi7NRqR3VY_10weavepy_vm
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys16build_with_state0B7_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17frame_locals_view
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8b_objectINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8b_object
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11call_object
+__RNvYNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types22install_object_dunders14object_setattrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types22install_object_dunders14object_setattr
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15weave_frame_mod7is_comp
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15weave_frame_mod15is_module_scope
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtCskHi7NRqR3VY_10weavepy_vm12shared_value7ThinArcNtNtBR_13tuple_storage12TupleStorageEE8grow_oneBR_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16invoke_finalizer
+__RNvCskHi7NRqR3VY_10weavepy_vm24generator_frame_traverse
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object8PyModuleE9drop_slowBK_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm6object8PyModuleEBF_
+__RNvMs12_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_10PyFunction5attrs
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5types12LeafAttrKindEBF_
+__RNvCskHi7NRqR3VY_10weavepy_vm15class_attr_fill
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins7set_add
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins17set_apply_inplace
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15core_local_attr
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21leaf_fused_local_attr
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins18set_membership_key
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_classs1_0EBJ_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24call_descriptor_accessor
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types20install_type_dunders12type_mro_get
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16rich_compare_obj
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21deep_equal_collection
+__RNvCskHi7NRqR3VY_10weavepy_vm10compare_op
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8set_copyINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5error19stop_iteration_with
+__RINvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types14make_exceptionReEB4_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter26fire_caught_stop_iteration
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real26make_ref_object_with_classs0_0B7_
+__RNvMs0_NtCskHi7NRqR3VY_10weavepy_vm16weakref_registryNtB5_11WeakRefSlot7upgrade
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins20generic_alias_origin
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11inline_bind
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11core_return
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14inline_deliver
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20park_suspended_boxed
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15core_gen_resume
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14try_inline_gen
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16leaf_load_method
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14core_gen_yield
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real11ref_type_eq
+__RNvMsh_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCs2NHf1dqD6Ob_4core6option6OptionReEE3getB7_
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockbE10initializeNCINvB2_11get_or_initNCNvCskHi7NRqR3VY_10weavepy_vm14fusion_enabled0E0zEB1w_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockbE10initializeNCINvB1a_11get_or_initNCNvCskHi7NRqR3VY_10weavepy_vm14fusion_enabled0E0zE0E0B2k_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15load_fast_value
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5error21attribute_error_named
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecPNtNtCskHi7NRqR3VY_10weavepy_vm5types10TypeObjectE8grow_oneBR_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21do_list_or_tuple_call
+__RNCINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB4_10str_encodeEs_0B6_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10str_encodeE9drop_slowBK_
+__RNCINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB4_9slot_callEs_0B6_
+__RNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB5_7GcState12collect_impl
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons24drain_pending_cext_drops
+__RNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB5_7GcState18collect_generation
+__RNvCskHi7NRqR3VY_10weavepy_vm13iter_traverse
+__RNvMs1q_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_10PyIterator12gc_referents
+__RNCNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB7_7GcState18collect_generations1_0B9_
+__RNvMs0_NtCskHi7NRqR3VY_10weavepy_vm16weakref_registryNtB5_11WeakRefSlot5clear
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8gc_trace19clear_object_fields
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7io_full5build
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io5build
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io19build_iobase_family
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io13iobase_method
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io19install_fileio_ctor0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io13make_buffered
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io13make_buffered0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io23install_delegating_attr
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io18make_memory_stream
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io18make_memory_stream0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io15set_type_module
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io32make_incremental_newline_decoder
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io32make_incremental_newline_decoder0B7_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2io8IoFamilyEBH_
+__RNCNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os22struct_seq_type_layout0se_0B9_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os9os_fspath
+__RNCINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB4_9str_splitEs_0B6_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8str_joinINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8str_join
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins18join_exact_strings
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15subscr_set_step
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter12store_subscr
+__RNvCskHi7NRqR3VY_10weavepy_vm22apply_slice_assignment
+__RNvCskHi7NRqR3VY_10weavepy_vm17slice_bound_index
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCsdq5mISKDh0s_5alloc3vec6splice6SpliceINtNtBG_9into_iter8IntoIterNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEEB1P_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20leaf_instance_dunder
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter25default_getattribute_slow
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types26make_exception_with_object
+__RNvMs1s_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object4repr
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object16bytes_repr_inner
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24build_exception_instance
+__RNvCskHi7NRqR3VY_10weavepy_vm21lookup_exception_init
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24instantiate_raised_class
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19normalize_exception
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter28break_implicit_context_cycle
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14sync_exc_attrs
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCskHi7NRqR3VY_10weavepy_vm5error11PyExceptionEEB1e_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm6object11PyTracebackEBF_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_rfindINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_rfind
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins19byte_offset_to_char
+__RNvMst_NtCskHi7NRqR3VY_10weavepy_vm12shared_valueNtB5_9SharedStr6repeat
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10str_rstripINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10str_rstrip
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins12str_endswithINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins12str_endswith
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter12do_next_call
+__RNvYFG0_RL1_SNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectRL0_STNtNtCsdq5mISKDh0s_5alloc6string6StringBb_EEINtNtCs2NHf1dqD6Ob_4core6result6ResultBb_NtNtBf_5error12RuntimeErrorEINtNtNtB1L_3ops8function2FnTRBa_RBX_EE4callBf_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os10os_stat_kw
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os10dir_fd_arg
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os14path_to_string
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8str_findINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNCINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB4_15bytes_decode_kwEs_0B6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_lower
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12unicode_case5lower
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os15stat_seq_finish
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter7repr_of
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11list_extendINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMsc_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10TypeObject6lookup
+__RNvCskHi7NRqR3VY_10weavepy_vm17resolve_metaclass
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21invoke_set_name_hooks
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11import_star
+__RNvXs2J_NtNtCs2NHf1dqD6Ob_4core5slice4iterINtB6_4IterNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectENtNtNtNtBa_4iter6traits8iterator8Iterator9size_hintBT_
+__RNvMs12_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_10PyFunction8set_slot
+__RNvCskHi7NRqR3VY_10weavepy_vm15instance_method
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5error21io_error_to_py_named2
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5error24set_oserror_filename_obj
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter29leaf_resolve_instance_attr_ix
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCskHi7NRqR3VY_10weavepy_vm8LeafAttrEBD_
+__RNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB7_11Interpreter21specialized_load_attr0B7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8str_find
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10bytes_data
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15bytes_decode_kwE9drop_slowBK_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os21stat_result_from_meta
+__RNvYNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types22install_object_dunders11object_initINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types22install_object_dunders11object_init
+__RNvMsi_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_11SlotStorage8index_of
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19run_py_exact_nofree
+__RNvMsj_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_10PyInstance11try_recycle
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14load_attr_type
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry15is_surface_only
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14core_pure_call
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_lowerINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11list_extend
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18is_data_descriptor
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os10os_listdir
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os9os_getgid
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter12format_value
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21core_pure_global_call
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib2os9os_getuid
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod31register_current_thread_pthread
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8dict_popINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12warnings_mod15default_filters
+__RNvMs3_NtCs4N3LIUsVkSw_17cranelift_codegen3isaDNtB5_9TargetIsaEL_15frontend_config
+__RNvMs2_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8functionNtB5_8Function19with_name_signature
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCs4N3LIUsVkSw_17cranelift_codegen3isa10IsaBuilderINtNtB4_6result6ResultINtNtCsdq5mISKDh0s_5alloc4sync3ArcDNtBE_9TargetIsaEL_ENtNtBG_6result12CodegenErrorEEECs4Ez510r8k3d_11weavepy_jit
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell4CellTyyEEE4withNCNvMNtNtBa_4hash6randomNtB1I_11RandomState3new0B21_ECs4Ez510r8k3d_11weavepy_jit
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze11infer_block
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapjmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE12contains_keyjECs4Ez510r8k3d_11weavepy_jit
+__RNCNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier212warm_compile0s2_0B7_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze10emit_block
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize26attempt_specialize_call_kw
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real26instance_method_trampoline
+__RNvNtCs4N3LIUsVkSw_17cranelift_codegen3isa6lookup
+__RNvXs1_NtCs4N3LIUsVkSw_17cranelift_codegen3isaINtB5_10IsaBuilderINtNtCs2NHf1dqD6Ob_4core6result6ResultINtNtCsdq5mISKDh0s_5alloc4sync3ArcDNtB5_9TargetIsaEL_ENtNtB7_6result12CodegenErrorEENtNtB7_8settings12Configurable6enableCsCBcs5KOPBz_16cranelift_native
+__RNvCsCBcs5KOPBz_16cranelift_native7builder
+__RNvCsCBcs5KOPBz_16cranelift_native18infer_native_flags
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit7analyze2SEE8grow_oneBQ_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7imp_mod5build
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7imp_mod7builtin
+__RNvMsn_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_10PyProperty6reinit
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins22construct_staticmethod
+__RNvMs_CskHi7NRqR3VY_10weavepy_vmNtB4_5Frame3top
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize34attempt_specialize_unpack_sequence
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18do_isinstance_call
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21deopt_load_attr_taken
+__RNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB7_11Interpreter16module_shadowings0_0B7_
+__RNCINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB4_14str_replace_kwEs_0B6_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7imp_mod14imp_is_builtin
+__RNvNtCskHi7NRqR3VY_10weavepy_vm10specialize26kw_uncovered_have_defaults
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real1b
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real26instance_dunder_trampoline
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real14module_getattr
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11dict_remove
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19kw_names_bind_check
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCskHi7NRqR3VY_10weavepy_vm5tier28JitStateEE4withNCNvB1x_12warm_compile0uEB1z_
+__RNvXs1_NtCs4N3LIUsVkSw_17cranelift_codegen8settingsNtB5_7BuilderNtB5_12Configurable3set
+__RNvXs1_NtCs4N3LIUsVkSw_17cranelift_codegen8settingsNtB5_7BuilderNtB5_12Configurable6enable
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze12analyze_impl
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze12analyze_once
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze12plan_rewrite
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze12build_blocks
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRjECs4Ez510r8k3d_11weavepy_jit
+__RINvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB6_7HashMapjuNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE12contains_keyjECs4Ez510r8k3d_11weavepy_jit
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze16reachable_blocks
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRmECs4Ez510r8k3d_11weavepy_jit
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit7analyze5ESlotE8grow_oneBQ_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze14emit_attr_lane
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionIBC_NtNtCs4Ez510r8k3d_11weavepy_jit7analyze2SEEEEEB1C_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins23reentrant_load_attr_opt
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15property_setterINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins16property_payload
+__RNvMsn_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_10PyProperty4fdel
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14wstr_attr_dict
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter18ensure_module_spec
+__RNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB7_11Interpreter16bind_python_argss8_0B7_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter27specialized_unpack_sequence
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter25recursive_isinstance_type
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7imp_mod13imp_is_frozen
+__RINvMs3_NtCsdq5mISKDh0s_5alloc3stre7replaceReECskHi7NRqR3VY_10weavepy_vm
+__RNCNvMNtCskHi7NRqR3VY_10weavepy_vm13builtin_typesNtB4_12BuiltinTypes5builds4_0B6_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7imp_mod18imp_create_builtin
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real5build
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexINtNtNtNtB7_11collections4hash3map7HashMapyjEEE10initializeNCINvB2_11get_or_initNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod16pthread_registry0E0zEB2M_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTyjEE14reserve_rehashNCINvNtB8_3map11make_hasheryjNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8dict_pop
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20kw_names_fill_locals
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12warnings_mod5build
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12warnings_mod10builtin_kw
+__RNvYNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12warnings_mod5builds_0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceTRSNtNtBa_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1R_EEE9call_onceBa_
+__RNvMs3_NtCs4Ez510r8k3d_11weavepy_jit6engineNtB5_9JitEngine3new
+__RNvMs0_NtCs4N3LIUsVkSw_17cranelift_codegen8settingsNtB5_7Builder6lookup
+__RNvMsn_NtCs4N3LIUsVkSw_17cranelift_codegen8settingsNtB5_5Flags3new
+__RNvMNtCs3m1Fwy4cFPt_13cranelift_jit7backendNtB2_10JITBuilder8with_isa
+__RNvMs0_NtCs3m1Fwy4cFPt_13cranelift_jit7backendNtB5_9JITModule3new
+__RNvMNtCsiev0CUJrxya_18cranelift_frontend8frontendNtB2_22FunctionBuilderContext3new
+__RNvXs3_NtNtCs2NHf1dqD6Ob_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCs4Ez510r8k3d_11weavepy_jit
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze14compute_livein
+__RNvXsL_NtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB5_9CtorStateNtNtCs2NHf1dqD6Ob_4core7default7Default7default
+__RINvXNtNtCsdq5mISKDh0s_5alloc3vec14spec_from_elemINtNtCs2NHf1dqD6Ob_4core6option6OptionINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit7analyze2SEEENtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEB1B_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit7analyze2SEENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze13step_abstract
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs4Ez510r8k3d_11weavepy_jit
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15infer_load_attr
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB2_9PathArena5names
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier220attr_fingerprint_obj
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBG_6string6StringEECs4Ez510r8k3d_11weavepy_jit
+__RINvXNtNtCsdq5mISKDh0s_5alloc3vec14spec_from_elemINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs4Ez510r8k3d_11weavepy_jit2ir6TBlockENtB3_12SpecFromElem9from_elemNtNtB7_5alloc6GlobalEB1r_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze10emit_instr
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit2ir5TStmtE8grow_oneBQ_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit2ir12AttrSiteMetaE8grow_oneBQ_
+__RNvMs4_NtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB5_5ESlot8is_plain
+__RINvMs_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecmE8dedup_byNCNvMs5_B5_Bv_5dedup0ECs4Ez510r8k3d_11weavepy_jit
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4Ez510r8k3d_11weavepy_jit7analyze9CtorStateEBF_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm11stdlib_tree8contains
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins18construct_property
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins21property_init_members
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins13property_with
+__RNvMsn_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_10PyProperty4fset
+__RNvMsn_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_10PyProperty3doc
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19do_dict_update_call
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15dict_merge_from
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit7analyze8RawBlockEEB1c_
+__RNvYNtNtCs3m1Fwy4cFPt_13cranelift_jit7backend9JITModuleNtNtCs7uvW3w2TaMW_16cranelift_module6module6Module13clear_contextCs4Ez510r8k3d_11weavepy_jit
+__RNvMs3_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph5clear
+__RNvMs2_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8functionNtB5_8Function5clear
+__RNvMsz_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8IntoIterNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities4InstINtCsf2xUKSCCMOo_8smallvec8SmallVecANtNtB1b_15user_stack_maps17UserStackMapEntryj4_EE10dying_nextB1d_
+__RNvMsz_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8IntoIterNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8constant12ConstantDataNtNtB1b_8entities8ConstantE10dying_nextB1d_
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECsiev0CUJrxya_18cranelift_frontend
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsiev0CUJrxya_18cranelift_frontend
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7extfunc8AbiParamE8grow_oneBS_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfg15ValueDataPackedE8grow_oneBS_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst16CompiledCodeBaseNtNtB10_6buffer5FinalEEEB12_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfg9BlockDataE8grow_oneBS_
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockNtNtCsiev0CUJrxya_18cranelift_frontend3ssa12SSABlockDataE20resize_for_index_mutB1X_
+__RNvMs3_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_10EntityListNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE4pushB16_
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs4N3LIUsVkSw_17cranelift_codegen
+__RNvXs0_NtCsiev0CUJrxya_18cranelift_frontend8frontendNtB5_15FuncInstBuilderNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7builder15InstBuilderBase5build
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockNtNtB12_6layout9BlockNodeE20resize_for_index_mutB14_
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockNtNtCsiev0CUJrxya_18cranelift_frontend8frontend11BlockStatusE20resize_for_index_mutB1X_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir12instructions15InstructionDataE8grow_oneBS_
+__RNvMsb_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph17make_inst_results
+__RNvMs4_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir6layoutNtB5_6Layout11append_inst
+__RNvMs3_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_10EntityListNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE8as_sliceB16_
+__RNvMsb_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph35non_tail_call_or_try_call_signature
+__RNvMsc_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir12instructionsNtB5_15InstructionData12analyze_call
+__RNvXNvMsb_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB8_13DataFlowGraph17inst_result_typesNtB2_15InstResultTypesNtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits8iterator8Iterator4next
+__RNvMsf_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir12instructionsNtB5_17OperandConstraint7resolve
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities4InstNtNtB12_6layout8InstNodeE20resize_for_index_mutB14_
+__RNvMs0_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir6layoutNtB5_6Layout15assign_inst_seq
+__RNvXNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB2_5InstsINtNtNtCs2NHf1dqD6Ob_4core3ops5index5IndexNtNtB4_8entities4InstE5index
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir5types4TypeE8grow_oneBS_
+__RNvMs4_NtCsiev0CUJrxya_18cranelift_frontend8frontendNtB5_15FunctionBuilder11try_def_var
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtCsiev0CUJrxya_18cranelift_frontend8variable8VariableIBF_NtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockINtNtB4_13packed_option12PackedOptionNtB1Y_5ValueEEE20resize_for_index_mutB12_
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockINtNtB4_13packed_option12PackedOptionNtB10_5ValueEE20resize_for_index_mutCsiev0CUJrxya_18cranelift_frontend
+__RNvMs3_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_10EntityListNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE4pushCs4Ez510r8k3d_11weavepy_jit
+__RNvMs3_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_10EntityListNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE4growCs4Ez510r8k3d_11weavepy_jit
+__RNvMNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir12instructionsNtB2_9BlockCall5block
+__RNvMs1_NtCsiev0CUJrxya_18cranelift_frontend3ssaNtB5_10SSABuilder25declare_block_predecessor
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser16parse_dict_value
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser17expect_dict_colon
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCslXHHUCZmUMI_14weavepy_parser3ast4ExprEE8grow_oneB1s_
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler15compile_subdict
+__RNvCs9DEk322Hfll_16weavepy_compiler25expr_contains_typeparamfn
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser11parse_yield
+__RNvCs9DEk322Hfll_16weavepy_compiler31collect_inlined_comp_inner_free
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler23compile_slice_two_parts
+__RINvNvNtCs9DEk322Hfll_16weavepy_compiler6intern13constant_hash4hashNtNtNtCs6KxzipSu1yP_3std4hash6random13DefaultHasherEB6_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTymEE14reserve_rehashNCINvNtB8_3map11make_hasherymNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs9DEk322Hfll_16weavepy_compiler
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtCs2NHf1dqD6Ob_4core6option6OptionmEE8grow_oneCs9DEk322Hfll_16weavepy_compiler
+__RNvCs9DEk322Hfll_16weavepy_compiler18collect_comp_reads
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast13ComprehensionEECs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast13ComprehensionECs9DEk322Hfll_16weavepy_compiler
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler29collect_global_walrus_shallow
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringNtCs9DEk322Hfll_16weavepy_compiler7CompSymEE8grow_oneB1a_
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler21collect_names_shallow
+__RNvCs9DEk322Hfll_16weavepy_compiler12comp_inlines
+__RNvCs9DEk322Hfll_16weavepy_compiler20expr_yields_in_scope
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler14comp_own_cells
+__RNvCs9DEk322Hfll_16weavepy_compiler29collect_comp_scope_inner_free
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler12comp_symbols
+__RNvCs9DEk322Hfll_16weavepy_compiler28collect_target_names_ordered
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler27collect_nested_comp_symbols
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringINtNtCs2NHf1dqD6Ob_4core6option6OptionNtCs9DEk322Hfll_16weavepy_compiler7BindingEEE8grow_oneB1M_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmNtNtCs9DEk322Hfll_16weavepy_compiler8bytecode6OpCodeNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBS_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmNtNtCs9DEk322Hfll_16weavepy_compiler8bytecode6OpCodeEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvMNtCs67nvpeQgtOq_13weavepy_lexer7scannerNtB2_7Scanner19scan_fstring_extent
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVeccE8grow_oneCs67nvpeQgtOq_13weavepy_lexer
+__RNvMNtCs67nvpeQgtOq_13weavepy_lexer7scannerNtB2_7Scanner25scan_fstring_field_extent
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser17fstring_parts_for
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser24parse_fstring_body_inner
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser19parse_fstring_field
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser28scrub_fstring_field_comments
+__RINvMNtCs2NHf1dqD6Ob_4core3stre12trim_matchesNCNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtBM_6Parser19parse_fstring_field0EBO_
+__RNvXs1i_NtCs2NHf1dqD6Ob_4core3fmtReNtB6_7Display3fmtCslXHHUCZmUMI_14weavepy_parser
+__RNvNtCs67nvpeQgtOq_13weavepy_lexer7scanner8tokenize
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser16remap_expr_spans
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser17parse_import_from
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast7KeywordECs9DEk322Hfll_16weavepy_compiler
+__RINvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB6_8Compiler18with_attr_locationNCNvB2_18compile_expr_inners3_0EB6_
+__RNvNtCs9DEk322Hfll_16weavepy_compiler7ast_opt6int_of
+__RNvMNtCsdq5mISKDh0s_5alloc5sliceSm6repeatCs9DEk322Hfll_16weavepy_compiler
+__RNvNtCs9DEk322Hfll_16weavepy_compiler7ast_opt8make_str
+__RNvNtCs9DEk322Hfll_16weavepy_compiler7ast_opt15optimize_format
+__RNvCs9DEk322Hfll_16weavepy_compiler14constant_slice
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueTNtCs9DEk322Hfll_16weavepy_compiler8ConstantBC_BC_EEBE_
+__RNvXs2_NtNtCs2NHf1dqD6Ob_4core5slice3cmpNtNtCsdq5mISKDh0s_5alloc6string6StringINtB5_14SlicePartialEqBC_E17equal_same_lengthCs4Ez510r8k3d_11weavepy_jit
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort25insertion_sort_shift_leftTjjENvYB1m_NtNtBa_3cmp10PartialOrd2ltECs9DEk322Hfll_16weavepy_compiler
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser11parse_raise
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser16parse_else_block
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler15emit_super_attr
+__RNvNtCs9DEk322Hfll_16weavepy_compiler6intern9name_hash
+__RNvNtCskHi7NRqR3VY_10weavepy_vm11stdlib_tree18write_install_json
+__RINvXNtNtCs10OuM599Pl4_10serde_json5value3serNtB5_5ValueNtNtCskautBTia54x_10serde_core3ser9Serialize9serializeQINtNtB7_3ser10SerializerQINtNtCsdq5mISKDh0s_5alloc3vec3VechENtB1O_15PrettyFormatterEECskHi7NRqR3VY_10weavepy_vm
+__RNvXs1_NtCs10OuM599Pl4_10serde_json3serQINtB5_10SerializerQINtNtCsdq5mISKDh0s_5alloc3vec3VechENtB5_15PrettyFormatterENtNtCskautBTia54x_10serde_core3ser10Serializer13serialize_strCskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm11stdlib_tree12write_atomic
+__RINvNvXs0_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB8_8BTreeMappppENtNtCs2NHf1dqD6Ob_4core5clone5Clone5clone13clone_subtreeNtNtBe_6string6StringNtNtCs10OuM599Pl4_10serde_json5value5ValueNtNtBe_5alloc6GlobalECskHi7NRqR3VY_10weavepy_vm
+__RNvXs_NtNtCs10OuM599Pl4_10serde_json5value5indexeNtB4_5Index15index_or_insert
+__RINvMs3_NtCsdq5mISKDh0s_5alloc3stre7replacecECskHi7NRqR3VY_10weavepy_vm
+__RNvYNCNvMNtCs67nvpeQgtOq_13weavepy_lexer7scannerNtB7_7Scanner11scan_number0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceThEE9call_onceB9_
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser9parse_del
+__RNvMsK_Cs9DEk322Hfll_16weavepy_compilerNtB5_8Compiler14compile_delete
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14format_obj_str
+__RNvCskHi7NRqR3VY_10weavepy_vm20format_via_spec_impl
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter30register_source_with_linecache
+__RNCINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB4_14str_splitlinesEs_0B6_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14str_splitlinesE9drop_slowBK_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8list_popINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins8list_pop
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins19code_synthetic_attr
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtCs9DEk322Hfll_16weavepy_compiler10CodeObjectE9drop_slowBI_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtCs9DEk322Hfll_16weavepy_compiler13WireOverridesEEEB1y_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24run_interpreter_shutdown
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10atexit_mod17run_exit_handlers
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib16interpreters_mod8RegistryEE10initializeNCINvB2_11get_or_initNCNvB1o_8registry0E0zEB1s_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtNtBa_6poison5mutex5MutexNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib16interpreters_mod8RegistryEE10initializeNCINvB1a_11get_or_initNCNvB2b_8registry0E0zE0E0B2f_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtB1a_5error12RuntimeErrorEEEB1a_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB1a_6object6ObjectEEEB1a_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtB1a_6option6OptionINtNtCsdq5mISKDh0s_5alloc5boxed3BoxSNtNtNtCskHi7NRqR3VY_10weavepy_vm5types10type_cache5EntryEEEEB2J_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter23run_shutdown_finalizers
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13flush_streams
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16current_sys_attr
+__RNvMs1l_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6PyFile9is_closed
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtBV_6string6StringNtNtB7_6object6ObjectEEE6borrowB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15file_check_open
+__RNvMs1l_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6PyFile15flush_write_buf
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCskHi7NRqR3VY_10weavepy_vm16InterpreterFlagsECsdSCkLiPtoQj_11weavepy_cli
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCsdSCkLiPtoQj_11weavepy_cli12EnvOverridesEBD_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCsdSCkLiPtoQj_11weavepy_cli3CliEBD_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtB1a_6object6ObjectB2l_EEEEB1a_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm3gil8GilGuardEEEB2h_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyNtNtCskHi7NRqR3VY_10weavepy_vm6object18KeyComparisonStateEB19_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCskHi7NRqR3VY_10weavepy_vm5tier28JitStateEEB1J_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtCs9DEk322Hfll_16weavepy_compiler8ConstantEECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCs9DEk322Hfll_16weavepy_compiler8ConstantECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtCs9DEk322Hfll_16weavepy_compiler13WireOverridesEEECskHi7NRqR3VY_10weavepy_vm
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB1a_13vm_singletons13ThreadHandlesEEEB1a_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib16faulthandler_mod16RegisteredThreadEBH_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecONtB1a_11InterpreterEEEB1a_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapNtNtCsdq5mISKDh0s_5alloc6string6StringINtNtB19_6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEEEB3o_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCskHi7NRqR3VY_10weavepy_vm12shared_value7ThinArcNtNtB19_13tuple_storage12TupleStorageEEB19_
+__RNvXsh_CskHi7NRqR3VY_10weavepy_vmNtB5_9LeanFrameNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RNvMs1l_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6PyFile14get_extra_attr
+__RNvMs1l_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6PyFile5flush
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtCskHi7NRqR3VY_10weavepy_vm6object7DictKeyNtB1b_6ObjectEEEB1d_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCs9DEk322Hfll_16weavepy_compiler5VmExtECskHi7NRqR3VY_10weavepy_vm
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtB1a_6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons23FaulthandlerThreadGuardEEEB27_
+__RNvMs4_NtCskHi7NRqR3VY_10weavepy_vm8gc_traceNtB5_7GcState23finalization_candidates
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10file_flushINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10file_flush
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib16faulthandler_mod16note_thread_exit
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtNtCs6KxzipSu1yP_3std6thread9lifecycle6PacketlEE9drop_slowCsdSCkLiPtoQj_11weavepy_cli
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellNtNtB1a_6object14ReprGuardStateEEB1a_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtCsdq5mISKDh0s_5alloc3vec3VecTjNtNtCskHi7NRqR3VY_10weavepy_vm8builtins12SetInplaceOpEEEEB2j_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native5eager7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellTNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectB1H_EEEB1L_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapjNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEEB2q_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellNtNtB19_5trace15MonitoringToolsEEB19_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtNtNtBa_11collections4hash3set7HashSetNtNtCsdq5mISKDh0s_5alloc6string6StringEEEB19_
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtNtNtBa_11collections4hash3map7HashMapNtNtBa_4path7PathBufTNtNtCskHi7NRqR3VY_10weavepy_vm6import8DirStampINtNtCsdq5mISKDh0s_5alloc2rc2RcINtNtB1K_3set7HashSetNtNtNtBa_3ffi6os_str8OsStringEEEEEEB2K_
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtNtNtCs6KxzipSu1yP_3std11collections4hash3set7HashSetNtNtNtBM_3ffi6os_str8OsStringEE9drop_slowCskHi7NRqR3VY_10weavepy_vm
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer5guard
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer9emit_exit
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7extfunc9SignatureE8grow_oneBS_
+__RNvMs2_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_8ListPoolNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE7reallocCs4Ez510r8k3d_11weavepy_jit
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs4Ez510r8k3d_11weavepy_jit
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer10emit_block
+__RNvMs4_NtCsiev0CUJrxya_18cranelift_frontend8frontendNtB5_15FunctionBuilder11try_use_var
+__RNvMs1_NtCsiev0CUJrxya_18cranelift_frontend3ssaNtB5_10SSABuilder16use_var_nonlocal
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE8grow_oneBS_
+__RNvMs1_NtCsiev0CUJrxya_18cranelift_frontend3ssaNtB5_10SSABuilder17run_state_machine
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueNtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeEE8grow_oneB1M_
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer15list_helper_sig
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4Ez510r8k3d_11weavepy_jit2ir6TBlockEBF_
+__RNvMs4_NtCsiev0CUJrxya_18cranelift_frontend8frontendNtB5_15FunctionBuilder15seal_all_blocks
+__RNvMs1_NtCsiev0CUJrxya_18cranelift_frontend3ssaNtB5_10SSABuilder14seal_one_block
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCsiev0CUJrxya_18cranelift_frontend3ssa4CallE8grow_oneBQ_
+__RNvMsc_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph18remove_block_param
+__RNvMs2_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_8ListPoolNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE4freeB13_
+__RNvMs2_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_8ListPoolNtNtCsiev0CUJrxya_18cranelift_frontend8variable8VariableE4freeB11_
+__RNvMs4_NtCsiev0CUJrxya_18cranelift_frontend8frontendNtB5_15FunctionBuilder8finalize
+__RNvMs9_NtNtCsiev0CUJrxya_18cranelift_frontend8frontend10safepointsNtB5_16SafepointSpiller5clear
+__RNvMs7_NtNtCsiev0CUJrxya_18cranelift_frontend8frontend10safepointsNtB5_16LivenessAnalysis5clear
+__RNvMse_NtCs7uvW3w2TaMW_16cranelift_module6moduleNtB5_18ModuleDeclarations16declare_function
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRNtNtCsdq5mISKDh0s_5alloc6string6StringECs7uvW3w2TaMW_16cranelift_module
+__RNvXs3_NtNtCs2NHf1dqD6Ob_4core4hash3sipINtB5_6HasherNtB5_11Sip13RoundsENtB7_6Hasher5writeCs7uvW3w2TaMW_16cranelift_module
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCsdq5mISKDh0s_5alloc6string6StringNtNtCs7uvW3w2TaMW_16cranelift_module6module12FuncOrDataIdEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1s_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB1w_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs7uvW3w2TaMW_16cranelift_module6module19FunctionDeclarationE8grow_oneBQ_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner14grow_amortizedCs7uvW3w2TaMW_16cranelift_module
+__RNvXs1_NtCs3m1Fwy4cFPt_13cranelift_jit7backendNtB5_9JITModuleNtNtCs7uvW3w2TaMW_16cranelift_module6module6Module34define_function_with_control_plane
+__RNvMNtCs4N3LIUsVkSw_17cranelift_codegen7contextNtB2_7Context15compile_stencil
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtCsdq5mISKDh0s_5alloc5boxed3BoxDNtNtCs4N3LIUsVkSw_17cranelift_codegen6timing8ProfilerEL_EEuE16get_or_init_slowNvNvNtB2r_7enabled8PROFILER27___rust_std_internal_init_fnEB2t_
+__RNvXs1_NtNtCs4N3LIUsVkSw_17cranelift_codegen6timing7enabledNtB7_15DefaultProfilerNtB7_8Profiler10start_pass
+__RNvMNtCs4N3LIUsVkSw_17cranelift_codegen7contextNtB2_7Context8optimize
+__RNvMNtCs4N3LIUsVkSw_17cranelift_codegen7contextNtB2_7Context8legalize
+__RNvXs0_NtCs4N3LIUsVkSw_17cranelift_codegen6cursorQNtB5_10FuncCursorNtNtNtB7_2ir7builder16InstInserterBase17insert_built_inst
+__RNvMs4_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir6layoutNtB5_6Layout11insert_inst
+__RNvXs_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB4_5InstsINtNtNtCs2NHf1dqD6Ob_4core3ops5index8IndexMutNtNtB6_8entities4InstE9index_mut
+__RNvMs_NtCs4N3LIUsVkSw_17cranelift_codegen9flowgraphNtB4_16ControlFlowGraph7compute
+__RNvMNtCs3vxoIrFW5GC_17cranelift_bforest4poolINtB2_8NodePoolINtNtB4_3map8MapTypesNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities4InstNtB1i_5BlockEE10alloc_nodeB1m_
+__RNvMs0_NtCs3vxoIrFW5GC_17cranelift_bforest4nodeINtB5_8NodeDataINtNtB7_3set8SetTypesNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockEE15try_leaf_insertB1p_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxDNtNtB4_3any3AnyEL_EECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen6timing7enabled18DefaultTimingTokenEBH_
+__RNvXs2_NtNtCs4N3LIUsVkSw_17cranelift_codegen6timing7enabledNtB5_18DefaultTimingTokenNtNtNtCs2NHf1dqD6Ob_4core3ops4drop4Drop4drop
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtNtCs4N3LIUsVkSw_17cranelift_codegen6timing7enabled9PassTimesEzE16get_or_init_slowNvNvB1R_9PASS_TIME27___rust_std_internal_init_fnEB1V_
+__RNvMs2_NtCs4N3LIUsVkSw_17cranelift_codegen14dominator_treeNtB5_13DominatorTree7compute
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4N3LIUsVkSw_17cranelift_codegen14dominator_tree14TraversalEventE8grow_oneBQ_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockE8grow_oneBS_
+__RNvMs_NtCs3vxoIrFW5GC_17cranelift_bforest4pathINtB4_4PathINtNtB6_3map8MapTypesNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities4InstNtB1g_5BlockEE5firstB1k_
+__RNvMs_NtCs3vxoIrFW5GC_17cranelift_bforest4pathINtB4_4PathINtNtB6_3map8MapTypesNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities4InstNtB1g_5BlockEE9next_nodeB1k_
+__RNvNtCs4N3LIUsVkSw_17cranelift_codegen16unreachable_code26eliminate_unreachable_code
+__RNvMs0_NtCseiVxvPL0Apu_16cranelift_bitset8compoundNtB5_14CompoundBitSet19try_ensure_capacityCs4N3LIUsVkSw_17cranelift_codegen
+__RNvNtCs4N3LIUsVkSw_17cranelift_codegen20remove_constant_phis23do_remove_constant_phis
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockNtNtB14_20remove_constant_phis12BlockSummaryE20resize_for_index_mutB14_
+__RNvMs6_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph19resolve_all_aliases
+__RNvMsA_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir12instructionsNtB5_15InstructionData13arguments_mut
+__RNvMs2_NtCs4N3LIUsVkSw_17cranelift_codegen13loop_analysisNtB5_12LoopAnalysis7compute
+__RNvMs_NtCs4N3LIUsVkSw_17cranelift_codegen14alias_analysisNtB4_13AliasAnalysis3new
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockuNtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE6insertBT_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBW_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockNtNtBW_14alias_analysis10LastStoresEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1L_NtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBW_
+__RNvMNtCs4N3LIUsVkSw_17cranelift_codegen14alias_analysisNtB2_10LastStores6update
+__RNCNvMs_NtCs4N3LIUsVkSw_17cranelift_codegen14alias_analysisNtB6_13AliasAnalysis26compute_block_input_statess_0B8_
+__RNvMs0_NtCs4N3LIUsVkSw_17cranelift_codegen6egraphNtB5_10EgraphPass3run
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueBY_E20resize_for_index_mutB14_
+__RNvNtCs4N3LIUsVkSw_17cranelift_codegen15inst_predicates18is_pure_for_egraph
+__RINvNtNtCs4N3LIUsVkSw_17cranelift_codegen4opts14generated_code29constructor_simplify_skeletonNtB4_11IsleContextINtCsf2xUKSCCMOo_8smallvec8SmallVecANtB2_26SkeletonInstSimplificationj5_EEB6_
+__RNvMsA_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir12instructionsNtB5_15InstructionData9arguments
+__RNvMs_NtNtCs4N3LIUsVkSw_17cranelift_codegen6egraph4costNtB4_4Cost9of_opcode
+__RNvMNtCs4N3LIUsVkSw_17cranelift_codegen14alias_analysisNtB2_10LastStores14get_last_store
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapNtNtCs4N3LIUsVkSw_17cranelift_codegen14alias_analysis9MemoryLocTNtNtNtBR_2ir8entities4InstNtB1R_5ValueENtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE6insertBR_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCs4N3LIUsVkSw_17cranelift_codegen14alias_analysis9MemoryLocTNtNtNtBU_2ir8entities4InstNtB1U_5ValueEEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1R_NtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBU_
+__RNvMs_NtCs4N3LIUsVkSw_17cranelift_codegen6egraphNtB4_11OptimizeCtx17insert_pure_enode
+__RNvMsb_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph12ctrl_typevar
+__RINvNtNtCs4N3LIUsVkSw_17cranelift_codegen4opts14generated_code20constructor_simplifyNtB4_11IsleContextINtCsf2xUKSCCMOo_8smallvec8SmallVecANtNtNtB6_2ir8entities5Valuej5_EEB6_
+__RNvXs2_NtCs4N3LIUsVkSw_17cranelift_codegen4optsNtB5_16InstDataEtorIterNtNtB5_14generated_code11ContextIter4next
+__RNvMs6_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph9value_def
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueuEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_uNtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBW_
+__RINvXss_Csf2xUKSCCMOo_8smallvecINtB6_8SmallVecANtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5Valuej5_EINtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits7collect6ExtendBJ_E6extendINtNtB1R_6option6OptionBJ_EEBP_
+__RNvMs4_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir6layoutNtB5_6Layout11remove_inst
+__RNvMsA_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir12instructionsNtB5_15InstructionData2eq
+__RNvMsA_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir12instructionsNtB5_15InstructionData15typevar_operand
+__RNvXs5_NtCs4N3LIUsVkSw_17cranelift_codegen4optsNtB5_18MaybeUnaryEtorIterNtNtB5_14generated_code11ContextIter4next
+__RINvNtNtCs4N3LIUsVkSw_17cranelift_codegen4opts14generated_code18constructor_truthyNtB4_11IsleContextINtB2_18ContextIterWrapperINtCsf2xUKSCCMOo_8smallvec8SmallVecANtNtNtB6_2ir8entities5Valuej8_EB1i_EEB6_
+__RNvMs0_NtNtCs4N3LIUsVkSw_17cranelift_codegen6egraph9elaborateNtB5_10Elaborator9elaborate
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtCs4N3LIUsVkSw_17cranelift_codegen10traversals5EventNtNtNtBR_2ir8entities5BlockEE8grow_oneBR_
+__RNvXs_NtCs4N3LIUsVkSw_17cranelift_codegen10traversalsNtB4_7DfsIterNtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits8iterator8Iterator4next
+__RNvMsb_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph11inst_values
+__RNvXs_NtNtNtCs2NHf1dqD6Ob_4core4iter8adapters5chainINtB4_5ChainIBO_INtNtB6_6copied6CopiedINtNtNtBa_5slice4iter4IterNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueEEINtNtB6_7flatten7FlatMapIB1q_NtNtB1T_12instructions9BlockCallEINtNtB6_10filter_map9FilterMapINtNtB6_3map3MapB1p_NCNvMB3h_B3f_4args0ENCNCNvMsb_NtB1T_3dfgNtB54_13DataFlowGraph11inst_values00ENCB4Y_0EEIB2N_INtNtBa_6option8IntoIterNtB1R_14ExceptionTableEIB3N_IB1q_NtNtB1T_15exception_table26InternalExceptionTableItemENCNvMB70_NtB70_18ExceptionTableData8contexts0ENCB4Y_s_0EENtNtNtB8_6traits8iterator8Iterator9size_hintB1V_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen6egraph9elaborate15BlockStackEntryE8grow_oneBS_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen6egraph9elaborate14ElabStackEntryE8grow_oneBS_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen6egraph9elaborate15ElaboratedValueE8grow_oneBS_
+__RNvMs0_NtNtCs4N3LIUsVkSw_17cranelift_codegen6egraph9elaborateNtB5_10Elaborator15maybe_remat_arg
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockNtBT_5ValueEB1M_EE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1M_NtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBX_
+__RNvMsb_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir3dfgNtB5_13DataFlowGraph10clone_inst
+__RNvMsA_NtNtCs4N3LIUsVkSw_17cranelift_codegen2ir12instructionsNtB5_15InstructionData10deep_clone
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueNtNtNtB14_6egraph9elaborate9BestEntryE20resize_for_index_mutB14_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen6egraph9elaborate10ElaboratorEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4N3LIUsVkSw_17cranelift_codegen14alias_analysis13AliasAnalysisEBF_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7extfunc9SignatureNtNtNtBW_8machinst3abi3SigEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1O_NtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBW_
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities6SigRefINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtNtB14_8machinst3abi3SigEE20resize_for_index_mutB14_
+__RNvMs5_NtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst3abiNtB5_6SigSet21abi_sig_for_signature
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4N3LIUsVkSw_17cranelift_codegen6result12CodegenErrorEBF_
+__RNvNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst3abi36ensure_struct_return_ptr_is_returned
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlocklE20resize_for_index_mutB14_
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockINtNtNtCs2NHf1dqD6Ob_4core3ops5range5RangejEE20resize_for_index_mutB14_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst10blockorder12LoweredBlockE8grow_oneBS_
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5BlockNtNtCsjhY5xleTbs_9regalloc25index5BlockE20resize_for_index_mutB14_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst10blockorder12LoweredBlockNtNtCsjhY5xleTbs_9regalloc25index5BlockEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B22_NtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBW_
+__RNvMs8_NtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst5vcodeNtB5_14VCodeConstants13with_capacity
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueINtNtNtB14_8machinst9valueregs9ValueRegsNtNtB1Y_3reg3RegEE20resize_for_index_mutB14_
+__RNvNtCs4N3LIUsVkSw_17cranelift_codegen15inst_predicates24has_lowering_side_effect
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities4InstNtNtNtBW_8machinst5lower9InstColorEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1K_NtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBW_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities4InstyEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_yNtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBW_
+__RINvNtNtNtCs2NHf1dqD6Ob_4core4iter8adapters5chain17and_then_or_clearINtB2_5ChainINtNtB4_6copied6CopiedINtNtNtB8_5slice4iter4IterNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueEEINtNtB4_7flatten7FlatMapIB1D_NtNtB26_12instructions9BlockCallEINtNtB4_10filter_map9FilterMapINtNtB4_3map3MapB1C_NCNvMB3u_B3s_4args0ENCNCNvMsb_NtB26_3dfgNtB5h_13DataFlowGraph11inst_values00ENCB5b_0EEB22_NvYB14_NtNtNtB6_6traits8iterator8Iterator4nextEB28_
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueNtNtNtB14_8machinst5lower13ValueUseStateE20resize_for_index_mutB14_
+__RNvMs3_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_10EntityListNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE3getB16_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecAINtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst9valueregs9ValueRegsNtNtBN_3reg3RegEj2_E8try_growBP_
+__RNvMs5_NtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst3abiNtB5_6SigSet11get_ret_arg
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst3abi7ArgPairE8grow_oneBS_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtCsjhY5xleTbs_9regalloc25index4InstNtBU_7PRegSetEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1s_NtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0ECs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCsjhY5xleTbs_9regalloc25index5BlockE8grow_oneCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecmE8grow_oneCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures8VRegDataE8grow_oneBS_
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCsjhY5xleTbs_9regalloc2
+__RNvMs4_NtNtCsdq5mISKDh0s_5alloc11collections9vec_dequeINtB5_8VecDequeNtNtCsjhY5xleTbs_9regalloc25index5BlockE4growCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs0_NtCsjhY5xleTbs_9regalloc28indexsetNtB5_8IndexSet10union_with
+__RNvMs0_NtCsjhY5xleTbs_9regalloc28indexsetNtB5_8IndexSet8is_empty
+__RNvMs6_NtNtCsjhY5xleTbs_9regalloc23ion15data_structuresNtB5_10LiveRanges3add
+__RNvXsw_CsjhY5xleTbs_9regalloc2NtB5_4BumpNtNtNtCs15PoI3sVs1y_14allocator_api26stable5alloc9Allocator8allocate
+__RNvMs7_CsiyzEERf8E16_7bumpaloNtB5_4Bump17alloc_layout_slowCsjhY5xleTbs_9regalloc2
+__RNvMsi_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8BTreeMapNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures12LiveRangeKeyNtB19_14LiveRangeIndexE6insertCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures18MultiFixedRegFixupE8grow_oneCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs7_CsiyzEERf8E16_7bumpaloNtB5_4Bump4growCsjhY5xleTbs_9regalloc2
+__RNvMs_NtNtCs15PoI3sVs1y_14allocator_api26stable3vecINtB4_3VecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures18LiveRangeListEntryNtB13_4BumpE6insertCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures14PrioQueueEntryE8grow_oneCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs_NtNtCsjhY5xleTbs_9regalloc23ion13reg_traversalNtB4_16RegTraversalIter3new
+__RNvXs0_NtNtCsjhY5xleTbs_9regalloc23ion13reg_traversalNtB5_16RegTraversalIterNtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits8iterator8Iterator4next
+__RINvMsi_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB6_8BTreeMapNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures12LiveRangeKeyNtB1a_14LiveRangeIndexE5rangeB18_INtNtNtCs2NHf1dqD6Ob_4core3ops5range9RangeFromB18_EECs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures15LiveBundleIndexE8grow_oneCs4N3LIUsVkSw_17cranelift_codegen
+__RINvMs_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree6removeINtNtB7_4node6HandleINtBY_7NodeRefNtNtBY_6marker3MutNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures12LiveRangeKeyNtB1N_14LiveRangeIndexNtB1v_4LeafENtB1v_2KVE14remove_leaf_kvNCNvMs5_NtNtB7_3map5entryINtB3T_13OccupiedEntryB1L_B2N_E9remove_kv0NtNtBb_5alloc6GlobalECs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures12InsertedMoveE8grow_oneBS_
+__RNvMNtCsjhY5xleTbs_9regalloc25movesINtB2_13ParallelMovesINtNtCs2NHf1dqD6Ob_4core6option6OptionNtB4_4VRegEE7resolveCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMNtNtCsjhY5xleTbs_9regalloc23ion15redundant_movesNtB2_23RedundantMoveEliminator12process_move
+__RNvMNtNtCsjhY5xleTbs_9regalloc23ion15redundant_movesNtB2_23RedundantMoveEliminator11clear_alloc
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsjhY5xleTbs_9regalloc23cfg7CFGInfoECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsjhY5xleTbs_9regalloc23cfg10CFGInfoCtxECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCsjhY5xleTbs_9regalloc28indexset8IndexSetEECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures10LiveRangesECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures11LiveBundlesECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures5VRegsECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures8PRegDataEECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures13SpillSlotDataEECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures14SpillSetRangesEECs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtCsiyzEERf8E16_7bumpalo4BumpE9drop_slowCsjhY5xleTbs_9regalloc2
+__RNvMse_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecANtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst6buffer9MachLabelj10_E6resizeBO_
+__RNvMNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst3regNtB2_3Reg5class
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst6buffer10MachBranchEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtCsf2xUKSCCMOo_8smallvec8IntoIterANtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst6buffer13MachLabelTrapj10_EEB1h_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst10blockorder18BlockLoweringOrderEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7extfunc8AbiParamEEB1e_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst5vcode14VCodeConstantsEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCs4hkhsQnToZM_16cranelift_entity7primary10PrimaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst5vcode13VCodeConstantNtB1y_17VCodeConstantDataEEB1C_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst3abi6SigSetEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst3abi6ABIArgEEB1e_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCsjhY5xleTbs_9regalloc26OutputECs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs_NtCs4N3LIUsVkSw_17cranelift_codegen8machinstINtB4_16CompiledCodeBaseNtNtB4_6buffer7StencilE12apply_params
+__RNvMs0_NtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst6bufferINtB5_19MachBufferFinalizedNtB5_7StencilE17apply_base_srcloc
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtCs7uvW3w2TaMW_16cranelift_module6module6FuncIdINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs3m1Fwy4cFPt_13cranelift_jit13compiled_blob12CompiledBlobEE20resize_for_index_mutB2s_
+__RINvNvMs2_NtCsdq5mISKDh0s_5alloc7raw_vecINtB8_11RawVecInnerpE7reserve21do_reserve_and_handleNtNtBa_5alloc6GlobalECs3m1Fwy4cFPt_13cranelift_jit
+__RNvMs5_NtCsdq5mISKDh0s_5alloc7raw_vecNtB5_11RawVecInner11finish_growCs3m1Fwy4cFPt_13cranelift_jit
+__RNvXs3_NtNtCs3m1Fwy4cFPt_13cranelift_jit6memory6systemNtB5_20SystemMemoryProviderNtB7_17JITMemoryProvider8allocate
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs3m1Fwy4cFPt_13cranelift_jit6memory6system6PtrLenE8grow_oneBS_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once9call_onceNCNvNtCs8ZRAllc6zij_6region4page4size0E0Cs3m1Fwy4cFPt_13cranelift_jit
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTjjNtNtCs7uvW3w2TaMW_16cranelift_module6module6FuncIdEE8grow_oneCs3m1Fwy4cFPt_13cranelift_jit
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs7uvW3w2TaMW_16cranelift_module6module6FuncIdE8grow_oneCs3m1Fwy4cFPt_13cranelift_jit
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7extfunc9SignatureEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecIBC_hEEECs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs0_NtCs3m1Fwy4cFPt_13cranelift_jit7backendNtB5_9JITModule20finalize_definitions
+__RNvXs3_NtNtCs3m1Fwy4cFPt_13cranelift_jit6memory6systemNtB5_20SystemMemoryProviderNtB7_17JITMemoryProvider8finalize
+__RNvMs0_NtCs3m1Fwy4cFPt_13cranelift_jit7backendNtB5_9JITModule22get_finalized_function
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir11GlobalGuardENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir12ListLoopMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir12IterLoopMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir13CompSavedMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir12AttrSiteMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir14MethodSiteMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir9StrMethodENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir13MathGuardMetaENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvXsb_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir8OsrEntryENtNtCs2NHf1dqD6Ob_4core5clone5Clone5cloneBJ_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit6engine14is_scalar_leaf
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4Ez510r8k3d_11weavepy_jit2ir5TFuncEBF_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir6TBlockEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir11GlobalGuardEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir12AttrSiteMetaEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4Ez510r8k3d_11weavepy_jit2ir12AttrSiteMetaEBF_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir14MethodSiteMetaEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir13MathGuardMetaEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir8OsrEntryEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4Ez510r8k3d_11weavepy_jit7analyze9PathArenaECskHi7NRqR3VY_10weavepy_vm
+__RNCNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier212warm_compile0s3_0B7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215attr_site_guard
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier215attr_site_guard0B5_
+__RNvMs6_NtCskHi7NRqR3VY_10weavepy_vm5tier2NtB5_8JitState16native_table_for
+__RNvMs6_NtCskHi7NRqR3VY_10weavepy_vm5tier2NtB5_8JitState23method_native_table_for
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_mod5build
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11marshal_mod8register
+__RNCINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB4_12int_to_bytesEs_0B6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins13byteorder_str
+__RNvNtNtCslSKxsPn8Dnr_10num_bigint7biguint7convert20to_bitwise_digits_le
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNvNtCskHi7NRqR3VY_10weavepy_vm8builtins12int_to_bytesE9drop_slowBK_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7imp_mod22imp_extension_suffixes
+__RNvCskHi7NRqR3VY_10weavepy_vm14spec_parent_of
+__RNvMs1w_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_6Object12interned_str
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtNtB5_6poison5mutex5MutexINtNtNtNtB7_11collections4hash3set7HashSetNtNtCskHi7NRqR3VY_10weavepy_vm12shared_value9SharedStrEEE10initializeNCINvB2_11get_or_initNCNvMs1w_NtB26_6objectNtB3D_6Object12interned_str0E0zEB26_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4time5build
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4time1b
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4time16compute_timezone
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4time16compute_timezone0B7_
+__RNvMNtNtCsej3qLsvwMv1_6chrono6offset3utcNtB2_3Utc3now
+__RNvMNtNtCsej3qLsvwMv1_6chrono5naive4dateNtB2_9NaiveDate25from_num_days_from_ce_opt
+__RNvMs15_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB6_11BoundMethod16is_pymethod_type
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10dict_clearINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10dict_clear
+__RNvCskHi7NRqR3VY_10weavepy_vm13bind_int_args
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11do_int_call
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins12b_int_compat
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins16parse_int_string
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins27transform_decimal_and_space
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4time17time_monotonic_ns
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtB7_4time7InstantE10initializeNCINvB2_11get_or_initNvMBV_BT_3nowE0zECskHi7NRqR3VY_10weavepy_vm
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtBc_4time7InstantE10initializeNCINvB1a_11get_or_initNvMB1I_B1G_3nowE0zE0E0CskHi7NRqR3VY_10weavepy_vm
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21percent_mod_left_slot
+__RNvCskHi7NRqR3VY_10weavepy_vm24percent_args_need_bridge
+__RNvCskHi7NRqR3VY_10weavepy_vm19percent_format_with
+__RINvMs3_NtCsdq5mISKDh0s_5alloc3stre7replaceAcj2_ECskHi7NRqR3VY_10weavepy_vm
+__RNvCskHi7NRqR3VY_10weavepy_vm10format_int
+__RNvCskHi7NRqR3VY_10weavepy_vm16group_int_digits
+__RNvCskHi7NRqR3VY_10weavepy_vm19int_precision_apply
+__RNvCskHi7NRqR3VY_10weavepy_vm9with_sign
+__RNvCskHi7NRqR3VY_10weavepy_vm15apply_alignment
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter8do_print
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4math5build
+__RNvCskHi7NRqR3VY_10weavepy_vm20checked_repeat_count
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4math7builtin
+__RINvCskHi7NRqR3VY_10weavepy_vm13try_alloc_vecNtNtB2_6object6ObjectEB2_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze17const_slice_shape
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15bin_result_type
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins7b_bytesINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib14operator_accel5build
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10b_bytes_kw
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13itertools_mod5build
+__RNvYNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13itertools_mod31register_classmethod_descriptorINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib14operator_accel5build0B7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins21bytes_from_source_obj
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib14datetime_accel5build
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13itertools_mod31register_classmethod_descriptor
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs4Ez510r8k3d_11weavepy_jit2ir6TBlockEEEB1y_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6result6ResultNtNtCskHi7NRqR3VY_10weavepy_vm6object10PyIteratorNtNtB11_5error12RuntimeErrorEEB11_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins29bytes_from_iterable_reentrant
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib14datetime_accel15timedelta_parts
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15byte_item_value
+__RNvCskHi7NRqR3VY_10weavepy_vm9float_pow
+__RINvCskHi7NRqR3VY_10weavepy_vm13try_alloc_vechEB2_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCs4Ez510r8k3d_11weavepy_jit2ir14MethodSiteMetaEEEB1y_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm14descr_registry10descr_type
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm5types12InstancePlanE9drop_slowBK_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11b_enumerateINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11b_enumerate
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins22coexist_builtin_dunder
+__RNvCskHi7NRqR3VY_10weavepy_vm16class_dunder_new
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell4CelljEE4withNCNvCskHi7NRqR3VY_10weavepy_vm13is_object_new0jEB1E_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmNtNtCs4Ez510r8k3d_11weavepy_jit2ir14ResolvedGlobalEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvCskHi7NRqR3VY_10weavepy_vm17is_union_eligible
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter22deopt_store_attr_taken
+__RNvCskHi7NRqR3VY_10weavepy_vm18is_union_initiator
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4Ez510r8k3d_11weavepy_jit7analyze4PlanEBF_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib14datetime_accel11date_fields
+__RNvCskHi7NRqR3VY_10weavepy_vm17make_pep604_union
+__RNCNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier212warm_compile0s6_0B7_
+__RNCNvCskHi7NRqR3VY_10weavepy_vm17make_pep604_union0B3_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter25push_pending_caller_shell
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier216probe_param_lane
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib14datetime_accel11time_fields
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze7pop_val
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native7install
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter12do_hash_call
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze9lower_bin
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier219resolve_plain_dicts
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11hash_object
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215classify_global
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRThReEECskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTxNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE8grow_oneBS_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier216probe_class_ctor
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTThReENtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_BV_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBZ_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTxNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEEB1e_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier222probe_class_ctor_shape
+__RNvMs2_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_nativeNtB5_5Names3new
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjTmmENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4Ez510r8k3d_11weavepy_jit
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native5build
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native5build0B7_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjTmmEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4Ez510r8k3d_11weavepy_jit
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjTmmmENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4Ez510r8k3d_11weavepy_jit
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer14div_guard_cond
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13leaf_builtins4push
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjTmmmEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4Ez510r8k3d_11weavepy_jit
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4Ez510r8k3d_11weavepy_jit
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer18floor_needs_adjust
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjmEE14reserve_rehashNCINvNtB8_3map11make_hasherjmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4Ez510r8k3d_11weavepy_jit
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjNtNtCs4Ez510r8k3d_11weavepy_jit7analyze7PairSrcNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBS_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11do_sum_call
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_upperINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins22sum_small_int_sequence
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjNtNtCs4Ez510r8k3d_11weavepy_jit7analyze7PairSrcEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_upper
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit2ir12ListLoopMetaE8grow_oneBQ_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins20sum_numeric_sequence
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins19sum_exact_int_range
+__RNvMNtCseIGkHRfguhS_9hashbrown11rustc_entryINtNtB4_3map7HashMapmINtNtCsdq5mISKDh0s_5alloc3vec3VecjENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE11rustc_entryCs4Ez510r8k3d_11weavepy_jit
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12unicode_case5upper
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_titleINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmINtNtCsdq5mISKDh0s_5alloc3vec3VecjEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4Ez510r8k3d_11weavepy_jit
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_title
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12unicode_case5title
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCsdq5mISKDh0s_5alloc3vec3VecThReNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEE10initializeNCINvB2_11get_or_initNCNvB2_10try_insert0E0zEB1y_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockINtNtCsdq5mISKDh0s_5alloc3vec3VecThReNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEE10initializeNCINvB1a_11get_or_initNCNvB1a_10try_insert0E0zE0E0B2l_
+__RNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB7_11Interpreter21percent_mod_left_slot0B7_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit2ir11GlobalGuardE8grow_oneBQ_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringNtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeEE8grow_oneCskHi7NRqR3VY_10weavepy_vm
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtNtNtBa_11collections4hash3map7HashMapReNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE4withNCNvNtB1G_8builtins12vm_intrinsic0INtNtCs2NHf1dqD6Ob_4core6option6OptionB1C_EEB1G_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmTmNtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBU_
+__RNvXs7_NtCs4N3LIUsVkSw_17cranelift_codegen4optsNtB5_11IsleContextNtNtB5_14generated_code7Context19iconst_sextend_etor
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtNtNtBe_11collections4hash3map7HashMapReNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEuE16get_or_init_slowNvNvNvNtB20_8builtins12vm_intrinsic5TABLE27___rust_std_internal_init_fnEB20_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtBG_6string6StringNtNtCs4Ez510r8k3d_11weavepy_jit2ir12CtorFieldSrcEEECskHi7NRqR3VY_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmTmNtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBX_
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueNtB10_5BlockE20resize_for_index_mutB14_
+__RNCNvNvNtCskHi7NRqR3VY_10weavepy_vm8builtins12vm_intrinsic5build0B7_
+__RNvMNtCs4hkhsQnToZM_16cranelift_entity3mapINtB2_12SecondaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValuehE20resize_for_index_mutB14_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTReNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_BS_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBW_
+__RINvNtNtCs4N3LIUsVkSw_17cranelift_codegen4opts14generated_code25constructor_ty_shift_maskNtB4_11IsleContextEB6_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_countINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins9str_count
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecANtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5Valuej5_E8try_growBO_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins13b_set_tp_name
+__RINvNtNtCs4N3LIUsVkSw_17cranelift_codegen4opts14generated_code20constructor_iconst_uNtB4_11IsleContextEB6_
+__RINvNtNtCs4N3LIUsVkSw_17cranelift_codegen4opts14generated_code20constructor_iconst_sNtB4_11IsleContextEB6_
+__RINvXss_Csf2xUKSCCMOo_8smallvecINtB6_8SmallVecANtNtNtCs4N3LIUsVkSw_17cranelift_codegen4opts14generated_code26SkeletonInstSimplificationj5_EINtNtNtNtCs2NHf1dqD6Ob_4core4iter6traits7collect6ExtendBJ_E6extendINtNtB2m_6option6OptionBJ_EEBP_
+__RINvNtNtCs4N3LIUsVkSw_17cranelift_codegen4opts14generated_code24constructor_compose_icmpNtB4_11IsleContextEB6_
+__RNvMs2_NtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB5_9CtorState10note_store
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4math13make_checked10B7_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst3abi6ABIArgE8grow_oneBS_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4math6to_f64
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14coerce_f64_opt
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215probe_list_lane
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215probe_dict_lane
+__RNvNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4math14freshen_result7freshen
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4math7builtin0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib4math9math_sqrt
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native13plain_drop_ok
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze14boundary_stack
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable7ipnsortjNvYjNtNtB8_3cmp10PartialOrd2ltECs4Ez510r8k3d_11weavepy_jit
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze9cmp_check
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable9quicksort9quicksortjNvYjNtNtBa_3cmp10PartialOrd2ltECs4Ez510r8k3d_11weavepy_jit
+__RNvXNtCs2NHf1dqD6Ob_4core3anyNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native5StateNtB2_3Any7type_idBx_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native11verify_slow
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze12infer_append
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native6dt_add
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter12do_bool_call
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native9dt_fields
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmTNtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeBP_ENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBT_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native9td_fields
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter20native_value_payload
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmTNtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeBS_EEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBW_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13class_ns_load
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze10seed_entry
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native10valid_date
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14class_ns_store
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native8dt_shift
+__RNvMs_NtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB4_4Plan9hidden_at
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter9stringify
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native7ymd2ord
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native7ord2ymd
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze22resolve_dict_container
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native6new_dt
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native14instance_fixed
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit2ir14MethodSpanMetaE8grow_oneBQ_
+__RNvMsg_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_11SlotStorage11from_layout
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit2ir14CalleeSpanMetaE8grow_oneBQ_
+__RNvMs3_NtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB5_2SE8is_plain
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze22entry_unassigned_reads
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native6dt_sub
+__RNvMs2_NtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB5_9CtorState10field_lane
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellINtNtCsdq5mISKDh0s_5alloc5boxed3BoxDNtNtCs4N3LIUsVkSw_17cranelift_codegen6timing8ProfilerEL_EEEB2j_
+__RNvMs2_NtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB5_9CtorState12stored_field
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native6new_td
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface11unwrap_shim0B5_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjThmmEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4Ez510r8k3d_11weavepy_jit
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface11unwrap_shim14check_receiver
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native6td_mul
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4Ez510r8k3d_11weavepy_jit
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native6td_cmp
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze17dyn_attr_fallback
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmmEE14reserve_rehashNCINvNtB8_3map11make_hashermmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4Ez510r8k3d_11weavepy_jit
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins12dict_setitemINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxSNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEB1f_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins12dict_setitem
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native12date_weekday
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit2ir8OsrEntryE8grow_oneBQ_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native14date_toordinal
+__RNvYNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types22install_object_dunders16object_class_getINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir9sourceloc12RelSourceLocE8grow_oneBS_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native11date_fields
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4Ez510r8k3d_11weavepy_jit2ir13RangeLoopMetaE8grow_oneBQ_
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types22install_object_dunders16object_class_get
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12weakref_real14proxy_referent
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier214probe_obj_live
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter25materialize_class_mapping
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native12dt_isoformat
+__RINvYNtNtCsiev0CUJrxya_18cranelift_frontend8frontend15FuncInstBuilderNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7builder11InstBuilder4jumpRINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB19_12instructions8BlockArgEECskHi7NRqR3VY_10weavepy_vm
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native5tz_of
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native13format_offset
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native16dt_fromisoformat
+__RNvCskHi7NRqR3VY_10weavepy_vm14intersect_sets
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native9parse_hms
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21try_dict_from_mapping
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins6b_dictINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjNtNtCs4Ez510r8k3d_11weavepy_jit7analyze14CompHeaderPlanEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvMsK_NtCsdq5mISKDh0s_5alloc4syncINtB5_4WeakNtNtCskHi7NRqR3VY_10weavepy_vm5types10PyInstanceE7upgradeBL_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins6b_dict
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native11dt_strftime
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native16strftime_numeric
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib15datetime_native8date_sub
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer5truth
+__RINvYNtNtCsiev0CUJrxya_18cranelift_frontend8frontend15FuncInstBuilderNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7builder11InstBuilder4brifRINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB19_12instructions8BlockArgEB2b_ECs4Ez510r8k3d_11weavepy_jit
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecANtNtCsjhY5xleTbs_9regalloc25index5Blockj10_E21reserve_one_uncheckedCs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm5tier211MethodEntryEEB1c_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecANtNtCsjhY5xleTbs_9regalloc25index5Blockj10_E8try_growCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer14pin_helper_sig
+__RNvYNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types20install_type_dunders12type_delattrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecANtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst6buffer10MachBranchj4_E21reserve_one_uncheckedBO_
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types20install_type_dunders12type_delattr
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer14stage_elements
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons17queue_parked_drop
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer15quad_helper_sig
+__RNvMs2_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_8ListPoolNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE7reallocB13_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapjThjENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertCs4Ez510r8k3d_11weavepy_jit
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTjThjEEE14reserve_rehashNCINvNtB8_3map11make_hasherjBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4Ez510r8k3d_11weavepy_jit
+__RNCNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier212warm_compile00B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native12deque_append
+__RNvMsi_NtCskHi7NRqR3VY_10weavepy_vm5typesNtB5_11SlotStorage14get_hinted_mut
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcDNtNtCs4N3LIUsVkSw_17cranelift_codegen3isa9TargetIsaEL_E9drop_slowCs3m1Fwy4cFPt_13cranelift_jit
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native13deque_popleft
+__RINvYNtNtCsiev0CUJrxya_18cranelift_frontend8frontend15FuncInstBuilderNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7builder11InstBuilder4brifRINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB19_12instructions8BlockArgERAB2J_j0_ECs4Ez510r8k3d_11weavepy_jit
+__RNvYNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7builder14ReplaceBuilderNtB4_11InstBuilder4iaddB8_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native7head_of
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsahm1S0TGo59_14target_lexicon6triple6TripleECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc4sync4WeakDNtNtCs4N3LIUsVkSw_17cranelift_codegen3isa9TargetIsaEL_RNtNtBG_5alloc6GlobalEECs3m1Fwy4cFPt_13cranelift_jit
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10b_callableINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtBG_5boxed3BoxDG_INtNtNtB4_3ops8function2FnTRL0_eEEp6OutputINtNtB4_6option6OptionPhENtNtB4_6marker4SendEL_EEECskHi7NRqR3VY_10weavepy_vm
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native13deque_getitem
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins18object_is_callable
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins4b_idINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxDG_INtNtNtB4_3ops8function2FnTRL0_eEEp6OutputINtNtB4_6option6OptionPhENtNtB4_6marker4SendEL_EECskHi7NRqR3VY_10weavepy_vm
+__RNvMs0_NtCs3vxoIrFW5GC_17cranelift_bforest4nodeINtB5_8NodeDataINtNtB7_3map8MapTypesNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities4InstNtB1l_5BlockEE15try_leaf_insertB1p_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins19coerce_index_object
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native8receiver
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native10DequeStateEBH_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins15object_identity
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native10deque_bool
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs3m1Fwy4cFPt_13cranelift_jit6memory6system20SystemMemoryProviderEBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs3m1Fwy4cFPt_13cranelift_jit6memory6system6MemoryEBH_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueNtNtBW_20remove_constant_phis13AbstractValueEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1L_NtCse8Yum7N1ehu_10rustc_hash13FxBuildHasherE0EBW_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze22resolve_list_container
+__RNvMs1_NtCs9DEk322Hfll_16weavepy_compiler8bytecodeNtB5_6OpCode4name
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs7uvW3w2TaMW_16cranelift_module6module18ModuleDeclarationsECskHi7NRqR3VY_10weavepy_vm
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native20deque_iterator_index
+__RNvMNtCseIGkHRfguhS_9hashbrown11rustc_entryINtNtB4_3map7HashMapmmNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE11rustc_entryCs4Ez510r8k3d_11weavepy_jit
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins16coerce_index_i64
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins20try_coerce_index_i64
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native19deque_iterator_next
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native10deque_next
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm6object14SlotDescriptorE9drop_slowBK_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter9op_binary
+__RNvMs_NtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB4_4Plan18comp_lane_for_body
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native16deque_appendleft
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm5types10TypeObjectE9drop_slowBK_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtNtCsdq5mISKDh0s_5alloc3vec6splice6SpliceINtNtNtNtB4_4iter7sources8repeat_n7RepeatNNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEEB24_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11do_len_call
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCs4hkhsQnToZM_16cranelift_entity3map12SecondaryMapNtNtCs7uvW3w2TaMW_16cranelift_module6module6DataIdINtNtB4_6option6OptionNtNtCs3m1Fwy4cFPt_13cranelift_jit13compiled_blob12CompiledBlobEEECskHi7NRqR3VY_10weavepy_vm
+__RNvMs3_NtCs4hkhsQnToZM_16cranelift_entity4listINtB5_10EntityListNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities5ValueE4growB16_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib18collections_native9deque_len
+__RNCNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier212warm_compile0s0_0B7_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter16core_pure_method
+__RNvCskHi7NRqR3VY_10weavepy_vm17coerce_len_result
+__RNCNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier212warm_compile0s1_0B7_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCs4N3LIUsVkSw_17cranelift_codegen13loop_analysis8LoopDataE8grow_oneBQ_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4N3LIUsVkSw_17cranelift_codegen7context7ContextECskHi7NRqR3VY_10weavepy_vm
+__RNCNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier212warm_compile0s7_0B7_
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmNtNtCs4Ez510r8k3d_11weavepy_jit2ir13CompSavedMetaNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertBS_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmNtNtCs4Ez510r8k3d_11weavepy_jit2ir13CompSavedMetaEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EBV_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze14escape_verdict
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8constant12ConstantPoolECskHi7NRqR3VY_10weavepy_vm
+__RNvMsz_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree3mapINtB5_8IntoIterNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8constant12ConstantDataNtNtB1b_8entities8ConstantE10dying_nextCskHi7NRqR3VY_10weavepy_vm
+__RNvNtCsjhY5xleTbs_9regalloc27domtree10merge_sets
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCs4hkhsQnToZM_16cranelift_entity7primary10PrimaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities9ImmediateNtNtB1A_8constant12ConstantDataEECskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures13BlockparamOutE8grow_oneCs4N3LIUsVkSw_17cranelift_codegen
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures12BlockparamInE8grow_oneCs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCs4hkhsQnToZM_16cranelift_entity7primary10PrimaryMapNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8entities14ExceptionTableNtNtB1A_15exception_table18ExceptionTableDataEECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir6layout6LayoutECskHi7NRqR3VY_10weavepy_vm
+__RINvXs_NtCs4N3LIUsVkSw_17cranelift_codegen7ctxhashNtB5_7NullCtxINtB5_7CtxHashTNtNtNtB7_2ir5types4TypeNtNtB1i_12instructions15InstructionDataEE8ctx_hashNtCse8Yum7N1ehu_10rustc_hash8FxHasherEB7_
+__RNvXsc_NtNtCs2NHf1dqD6Ob_4core5slice5indexTINtNtNtB9_3ops5range5BoundjEBF_EINtB5_10SliceIndexSNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures18LiveRangeListEntryE5indexCs4N3LIUsVkSw_17cranelift_codegen
+__RNvYNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir7builder14ReplaceBuilderNtB4_11InstBuilder4bxorB8_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir10debug_tags9DebugTagsECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtNtCs4N3LIUsVkSw_17cranelift_codegen2ir8function18FunctionParametersECskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures13SpillSlotDataE8grow_oneCs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4N3LIUsVkSw_17cranelift_codegen9flowgraph16ControlFlowGraphECskHi7NRqR3VY_10weavepy_vm
+__RNCNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier212warm_compile0sa_0B7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier211guards_hold
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier28take_u64
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4N3LIUsVkSw_17cranelift_codegen14dominator_tree13DominatorTreeECskHi7NRqR3VY_10weavepy_vm
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCskHi7NRqR3VY_10weavepy_vm5tier27JitBufsEuE16get_or_init_slowNvNvB1R_8JIT_BUFS27___rust_std_internal_init_fnEB1T_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier28take_u32
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCskHi7NRqR3VY_10weavepy_vm5tier23PinE8grow_oneBQ_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4N3LIUsVkSw_17cranelift_codegen13loop_analysis12LoopAnalysisECskHi7NRqR3VY_10weavepy_vm
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCskHi7NRqR3VY_10weavepy_vm5tier28JitStateEE4withNCNvB1x_19current_compile_gen0yEB1z_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier214wpjit_attr_get
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier211unpack_pins
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionINtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst16CompiledCodeBaseNtNtB10_6buffer5FinalEEECskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier221drain_activation_pins
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCsiev0CUJrxya_18cranelift_frontend8frontend22FunctionBuilderContextECskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecyEE8grow_oneCskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecINtNtB7_3vec3VecmEE8grow_oneCs9DEk322Hfll_16weavepy_compiler
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5tier27CallCtxEBF_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm5tier23PinEEB1c_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecATmNtNtNtCs4N3LIUsVkSw_17cranelift_codegen3isa6unwind10UnwindInstEj8_E21reserve_one_uncheckedBQ_
+__RNvYNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13itertools_mod10count_coreINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13itertools_mod10count_core
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real13allocate_lock
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real16make_lock_object
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13functools_mod5build
+__RNCNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13functools_mod8lru_type00B9_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real16new_rlock_object
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtNtNtBa_11collections4hash3map7HashMapReNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEEB1Q_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter11do_str_call
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter12load_special
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys10sys_intern
+__RNCNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB7_11Interpreter12load_special0B7_
+__RINvMs10_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree4nodeINtB7_16BalancingContextNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures12LiveRangeKeyNtB1l_14LiveRangeIndexE25merge_tracking_child_edgeNtNtBd_5alloc6GlobalECs4N3LIUsVkSw_17cranelift_codegen
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real26instance_dunder_trampoline0B7_
+__RNvMs10_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree4nodeINtB6_16BalancingContextNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures12LiveRangeKeyNtB1k_14LiveRangeIndexE15bulk_steal_leftCs4N3LIUsVkSw_17cranelift_codegen
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real16make_lock_objects1_0B7_
+__RNvMsA_NtCskHi7NRqR3VY_10weavepy_vm4syncNtB5_8RealLock11try_acquire
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins16str_isidentifierINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier214entry_local_ok
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier220decompose_live_loops
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins16str_isidentifier
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real16make_lock_objects2_0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real17thread_handle_new
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins12obj_containsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real25make_thread_handle_object
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockINtNtCsl3bTtyQSXML_8lock_api5mutex5MutexNtNtCsfqmARTQr4sq_11parking_lot9raw_mutex8RawMutexINtNtNtNtB7_11collections4hash3map7HashMapyINtNtCsdq5mISKDh0s_5alloc4sync3ArcNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real17ThreadHandleStateEEEE10initializeNCINvB2_11get_or_initNCNvB3C_15handle_registry0E0zEB3G_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier214enter_compiled
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTyINtNtCsdq5mISKDh0s_5alloc4sync3ArcNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real17ThreadHandleStateEEE14reserve_rehashNCINvNtB8_3map11make_hasheryBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB1v_
+__RNvMs10_NtNtNtCsdq5mISKDh0s_5alloc11collections5btree4nodeINtB6_16BalancingContextNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures12LiveRangeKeyNtB1k_14LiveRangeIndexE16bulk_steal_rightCs4N3LIUsVkSw_17cranelift_codegen
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins24bytearray_receiver_guard
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real15new_lock_object
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object19current_interp_hash
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17merge_exec_kwargs
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter12do_eval_call
+__RNvCskHi7NRqR3VY_10weavepy_vm26check_compile_source_nulls
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtNtCskHi7NRqR3VY_10weavepy_vm5tier29ArtifactsE9drop_slowBH_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5tier29ArtifactsEBF_
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser10parse_eval
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtNtCs4Ez510r8k3d_11weavepy_jit6engine13CompiledFrameE9drop_slowCskHi7NRqR3VY_10weavepy_vm
+__RNvNtCslXHHUCZmUMI_14weavepy_parser6parser21parse_eval_with_flufl
+__RNCINvMs18_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB9_9BuiltinFn11with_kwargsNCNvNtNtBb_6stdlib11thread_real16make_lock_object0Es_0Bb_
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser17parse_eval_module
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real16make_lock_object0B7_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4Ez510r8k3d_11weavepy_jit6engine13CompiledFrameECskHi7NRqR3VY_10weavepy_vm
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real20resolve_acquire_args
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser18parse_eval_element
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier217wpjit_list_append
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier214wpjit_list_len
+__RNvMNtCslXHHUCZmUMI_14weavepy_parser6parserNtB2_6Parser12parse_lambda
+__RNvCslXHHUCZmUMI_14weavepy_parser21eval_backslash_at_eof
+__RNvNvCskHi7NRqR3VY_10weavepy_vm32collect_compiler_syntax_warnings10visit_body
+__RNvNvCskHi7NRqR3VY_10weavepy_vm32collect_compiler_syntax_warnings10visit_expr
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real21get_main_thread_ident
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real18make_thread_handle
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable14driftsort_mainTNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures11PosWithPrioNtB16_4EditENCINvMNtCsdq5mISKDh0s_5alloc5sliceSBZ_11sort_by_keyyNCNvMsp_B12_NtB12_5Edits4sort0E0INtNtB2l_3vec3VecBZ_EECs4N3LIUsVkSw_17cranelift_codegen
+__RNvNvCskHi7NRqR3VY_10weavepy_vm32collect_compiler_syntax_warnings10visit_args
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable5drift4sortTNtNtNtCsjhY5xleTbs_9regalloc23ion15data_structures11PosWithPrioNtB13_4EditENCINvMNtCsdq5mISKDh0s_5alloc5sliceSBW_11sort_by_keyyNCNvMsp_BZ_NtBZ_5Edits4sort0E0ECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs67nvpeQgtOq_13weavepy_lexer5token13EscapeWarningEECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real25make_thread_handle_object0EBJ_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter25exec_eval_compile_options
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real13get_native_id
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real17make_rlock_objects6_0B7_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecANtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst10blockorder12LoweredBlockj40_E8try_growBO_
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real17make_rlock_objects7_0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10os_process19register_at_fork_kw
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit2ir12AttrSiteMetaEECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecTmNtNtCskHi7NRqR3VY_10weavepy_vm5tier29SynthSeedEEEB1e_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5tier213CompiledEntryEBF_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze16infer_store_attr
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB14_6string6StringENtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeIBC_TB1w_mEEbEEEB1X_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecAhj400_E8try_growCs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast8StmtKindECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast9ArgumentsECskHi7NRqR3VY_10weavepy_vm
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtNtCskHi7NRqR3VY_10weavepy_vm5tier213GuardSnapshotE9drop_slowBH_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter14lazy_iter_next
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast3ArgEECskHi7NRqR3VY_10weavepy_vm
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCslXHHUCZmUMI_14weavepy_parser3ast3ArgECskHi7NRqR3VY_10weavepy_vm
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtB7_3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE9drop_slowBX_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCslXHHUCZmUMI_14weavepy_parser3ast4ExprEEECskHi7NRqR3VY_10weavepy_vm
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBW_6string6StringEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECs4Ez510r8k3d_11weavepy_jit
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtB7_3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm5tier29AttrGuardEE9drop_slowBX_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier214wpjit_attr_set
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm5tier29AttrGuardEEB1c_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast4ExprEECskHi7NRqR3VY_10weavepy_vm
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRTmNtNtCsdq5mISKDh0s_5alloc6string6StringEECs4Ez510r8k3d_11weavepy_jit
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtB7_3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm5tier211MethodEntryEE9drop_slowBX_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTTmNtNtCsdq5mISKDh0s_5alloc6string6StringENtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_B1v_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB1z_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCslXHHUCZmUMI_14weavepy_parser3ast4ExprEECskHi7NRqR3VY_10weavepy_vm
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecANtCsjhY5xleTbs_9regalloc210Allocationj4_E21reserve_one_uncheckedBK_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins16mappingproxy_getINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCslXHHUCZmUMI_14weavepy_parser3ast7KeywordEECskHi7NRqR3VY_10weavepy_vm
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins16mappingproxy_get
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins21mappingproxy_delegate
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins17mappingproxy_args
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcINtNtB7_3vec3VecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm5tier212NativeCalleeEEE9drop_slowB1z_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real9get_ident
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtB4_6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm5tier212NativeCalleeEEEB1y_
+__RNvYNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types20install_type_dunders12type_setattrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter30winner_aware_dynamic_type_call
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types20install_type_dunders12type_setattr
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins7set_popINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNCINvMs18_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB9_9BuiltinFn11with_kwargsNvNtNtBb_6stdlib11thread_real21start_joinable_threadEs_0Bb_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins7set_pop
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys16build_with_states_0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real19spawn_python_worker
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib3sys22sys_getframemodulename
+__RINvMNtNtCs6KxzipSu1yP_3std4sync9once_lockINtB3_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm15thread_registry14ThreadRegistryE10initializeNCINvB2_11get_or_initNvMs0_BV_BT_3newE0zEBX_
+__RNCINvMs0_NtNtCs6KxzipSu1yP_3std4sync4onceNtB8_4Once15call_once_forceNCINvMNtBa_9once_lockINtB1b_8OnceLockNtNtCskHi7NRqR3VY_10weavepy_vm15thread_registry14ThreadRegistryE10initializeNCINvB1a_11get_or_initNvMs0_B1I_B1G_3newE0zE0E0B1K_
+__RNvYNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13functools_mod12partial_callINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectRSTNtNtCsdq5mISKDh0s_5alloc6string6StringB1R_EEE4callB8_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib13functools_mod12partial_call
+__RNvMs0_NtCskHi7NRqR3VY_10weavepy_vm15thread_registryNtB5_14ThreadRegistry8register
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE8grow_oneB1c_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real26lock_acquire_interruptible
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10struct_mod5build
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types20install_type_dunders12type_doc_get
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable7ipnsortNtNvMNtNtCsjhY5xleTbs_9regalloc23ion5movesINtNtB10_15data_structures3EnvpE34apply_allocations_and_insert_moves14InterBlockDestNCINvMB6_SBT_20sort_unstable_by_keyyNvMs_BV_BT_3keyE0ECs4N3LIUsVkSw_17cranelift_codegen
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable9quicksort9quicksortNtNvMNtNtCsjhY5xleTbs_9regalloc23ion5movesINtNtB1e_15data_structures3EnvpE34apply_allocations_and_insert_moves14InterBlockDestNCINvMB8_SB17_20sort_unstable_by_keyyNvMs_B19_B17_3keyE0ECs4N3LIUsVkSw_17cranelift_codegen
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real16make_lock_objects_0B7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface20obj_contains_builtin
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10codecs_mod5build
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCskHi7NRqR3VY_10weavepy_vm3gil8GilGuardEEB1c_
+__RNvCskHi7NRqR3VY_10weavepy_vm25detect_yield_from_subiter
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm3gil8GilGuardEBF_
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface22install_object_compare6obj_eq
+__RNvMsA_NtCskHi7NRqR3VY_10weavepy_vm4syncNtB5_8RealLock15acquire_timeout
+__RNvMs_NtCsfqmARTQr4sq_11parking_lot7condvarNtB4_7Condvar19wait_until_internal
+__RINvMs0_NtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazyINtB6_7StorageNtNtCs8TNC6SO7etN_16parking_lot_core11parking_lot10ThreadDatauE16get_or_init_slowNvNvNvB1g_16with_thread_data11THREAD_DATA27___rust_std_internal_init_fnECsfqmARTQr4sq_11parking_lot
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215callee_ret_lane
+__RNvMs1_NtCs8TNC6SO7etN_16parking_lot_core11parking_lotNtB5_10ThreadData3new
+__RNvMNtCs8TNC6SO7etN_16parking_lot_core11parking_lotNtB2_9HashTable3new
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyNtNtCskHi7NRqR3VY_10weavepy_vm6object18KeyComparisonStateE4withNCINvBW_13key_cmp_scopeINtNtCs2NHf1dqD6Ob_4core6option6OptionNtBW_6ObjectENCNvMs3_BY_NtBY_11Interpreter11instantiates_0E0INtNtB2n_6result6ResultB2i_NtNtBY_5error12RuntimeErrorEEBY_
+__RNvMs_NtCskHi7NRqR3VY_10weavepy_vm6importNtB4_11ModuleCache6remove
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15analyze_for_ret
+__RNSNvYNCINvNtNtCs6KxzipSu1yP_3std6thread9lifecycle15spawn_uncheckedNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real19spawn_python_workers1_0uEs_0INtNtNtCs2NHf1dqD6Ob_4core3ops8function6FnOnceuE9call_once6vtableB1b_
+__RINvNtNtCs6KxzipSu1yP_3std3sys9backtrace28___rust_begin_short_backtraceNCNCNCINvNtNtB6_6thread9lifecycle15spawn_uncheckedNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real19spawn_python_workers1_0uEs_000uEB23_
+__RNvMs3_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_7PyFrame15locals_snapshot
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier215callee_ret_infos0_0B5_
+__RINvNtNtCs6KxzipSu1yP_3std3sys9backtrace28___rust_begin_short_backtraceNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real19spawn_python_workers1_0uEB1f_
+__RNCNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15analyze_for_rets5_0B5_
+__RNvMs3_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_7PyFrame22has_live_hidden_locals
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4Ez510r8k3d_11weavepy_jit7analyze9PathArenaEBF_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons24install_worker_thread_id
+__RNvMs3_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_7PyFrame14compute_locals
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable14driftsort_mainNtNtCsdq5mISKDh0s_5alloc6string6StringNvYBZ_NtNtB8_3cmp10PartialOrd2ltINtNtB13_3vec3VecBZ_EECs9DEk322Hfll_16weavepy_compiler
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable5drift4sortNtNtCsdq5mISKDh0s_5alloc6string6StringNvYBW_NtNtBa_3cmp10PartialOrd2ltECs9DEk322Hfll_16weavepy_compiler
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTyyEE14reserve_rehashNCINvNtB8_3map11make_hasheryyNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECskHi7NRqR3VY_10weavepy_vm
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6stable9quicksort9quicksortNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB15_NtNtBa_3cmp10PartialOrd2ltECs9DEk322Hfll_16weavepy_compiler
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons26install_seed_builtin_types
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared5pivot11median3_recNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB14_NtNtBa_3cmp10PartialOrd2ltECs9DEk322Hfll_16weavepy_compiler
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort31small_sort_general_with_scratchNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB1s_NtNtBa_3cmp10PartialOrd2ltECs9DEk322Hfll_16weavepy_compiler
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared9smallsort12sort4_stableNtNtCsdq5mISKDh0s_5alloc6string6StringNvYB19_NtNtBa_3cmp10PartialOrd2ltECs9DEk322Hfll_16weavepy_compiler
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecONtB7_11InterpreterEE21try_borrow_mut_lockedB7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB7_13vm_singletons13ThreadHandlesEE21try_borrow_mut_lockedB7_
+__RNCNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15analyze_for_rets_0B5_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11str_isupperINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNCNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15analyze_for_rets0_0B5_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11str_isupper
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtB7_6string6StringNtNtCs4Ez510r8k3d_11weavepy_jit2ir12CtorFieldSrcEE8grow_oneCskHi7NRqR3VY_10weavepy_vm
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object8DictDataE17try_borrow_lockedB7_
+__RNCNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15analyze_for_ret0B5_
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier215callee_ret_infos1_0B5_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBI_6string6StringEB1a_EECskHi7NRqR3VY_10weavepy_vm
+__RNCNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier212warm_compile0s4_0B7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_16weakref_registry13RegistryInnerE17try_borrow_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier218probe_method_entry
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtB7_6object6ObjectEE17try_borrow_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215method_ret_info
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object8DictDataE21try_borrow_mut_lockedB7_
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier215method_ret_infos_0B5_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB7_6object6ObjectEE17try_borrow_lockedB7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel5build
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB7_6object6ObjectEE21try_borrow_mut_lockedB7_
+__RNvYNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel10make_loadsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier215method_ret_infos8_0B5_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel10make_loads
+__RINvYNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateNtNtCs2NHf1dqD6Ob_4core4hash11BuildHasher8hash_oneRTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtB1M_6string6StringEB2e_EECskHi7NRqR3VY_10weavepy_vm
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_5types11SlotStorageE21try_borrow_mut_lockedB7_
+__RNvMNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accelNtB2_13FunctionGuard3new
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtBX_6string6StringEB1p_EmEE14reserve_rehashNCINvNtB8_3map11make_hasherBQ_mNtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0ECskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtCskHi7NRqR3VY_10weavepy_vm5tier211MethodEntryE8grow_oneBQ_
+__RNvMs_NtCsfqmARTQr4sq_11parking_lot7condvarNtB4_7Condvar15notify_one_slow
+__RNvNtCs8TNC6SO7etN_16parking_lot_core11parking_lot16lock_bucket_pair
+__RNvMs1_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accelNtB5_11ImportGuard3new
+__RNvMs1_NtCsfqmARTQr4sq_11parking_lot9raw_mutexNtB5_8RawMutex9lock_slow
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_5types11SlotStorageE17try_borrow_lockedB7_
+__RNvMs0_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accelNtB5_12BuiltinGuard3new
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13delete_subscr
+__RNvNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel7classes8pure_get
+__RNvMs_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accelNtB4_20ModuleFunctionsGuard3new
+__RNvMs6_NtCskHi7NRqR3VY_10weavepy_vm5tier2NtB5_8JitState21resolve_native_callee
+__RNvMs6_NtCskHi7NRqR3VY_10weavepy_vm5tier2NtB5_8JitState19resolve_native_func
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15pull_for_unpack
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier215method_ret_infos2_0B5_
+__RNvMs_NtCs4Ez510r8k3d_11weavepy_jit2irNtB4_9StrMethod9from_name
+__RNvYNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel10make_dumpsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel10make_dumps
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier215method_ret_infos6_0B5_
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier215method_ret_infos0_0B5_
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm5tier215method_ret_infosa_0B5_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellAjj3_E21try_borrow_mut_lockedB7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellAjj3_E17try_borrow_lockedB7_
+__RNvMs5_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accelNtB5_19ClassFunctionsGuard3new
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE21try_borrow_mut_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier220wpjit_iter_next_pair
+__RNvYNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types22install_object_dunders14object_delattrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB8_6object6ObjectEE4callB8_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215wpjit_bytes_get
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCs4Ez510r8k3d_11weavepy_jit2ir14MethodSiteMetaEE6resizeB1l_
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types22install_object_dunders14object_delattr
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecONtB7_11InterpreterEE17try_borrow_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier216wpjit_global_obj
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24generic_delattr_instance
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTmyEE8grow_oneCskHi7NRqR3VY_10weavepy_vm
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel19ClassFunctionsGuardE8grow_oneBS_
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit5lowerNtB2_7Lowerer22call_method_helper_sig
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier214wpjit_call_dyn
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter30run_thread_local_death_cleanup
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter24call_object_with_globals
+__RNvMNtCs4Ez510r8k3d_11weavepy_jit7analyzeNtB2_9PathArena3seg
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier27pin_any
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTINtNtCs2NHf1dqD6Ob_4core6option6OptionmEmNtNtB7_6string6StringEE8grow_oneCs4Ez510r8k3d_11weavepy_jit
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared5pivot11median3_recNtNvMNtNtCsjhY5xleTbs_9regalloc23ion5movesINtNtB1b_15data_structures3EnvpE34apply_allocations_and_insert_moves14InterBlockDestNCINvMB8_SB14_20sort_unstable_by_keyyNvMs_B16_B14_3keyE0ECs4N3LIUsVkSw_17cranelift_codegen
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215attr_chain_step
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_16weakref_registry13RegistryInnerE21try_borrow_mut_lockedB7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtB7_6object6ObjectB1o_EEE17try_borrow_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier214wpjit_list_set
+__RNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface15obj_len_builtin
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface19install_type_checks18type_subclasscheck
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtCskHi7NRqR3VY_10weavepy_vm11InterpreterEBD_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm6import11ModuleCacheEBF_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins18mappingproxy_itemsINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins18mappingproxy_items
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueTmNtNtCskHi7NRqR3VY_10weavepy_vm5tier29SynthSeedEEBH_
+__RNvCskHi7NRqR3VY_10weavepy_vm16fallback_globals
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtCskHi7NRqR3VY_10weavepy_vm9InlineActEEB1c_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7sre_mod5build
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtCskHi7NRqR3VY_10weavepy_vm12shared_value7ThinArcNtNtB1d_13tuple_storage12TupleStorageEEEB1d_
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types14make_owned_news_0B5_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object18native_scalar_repr
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types14make_owned_new11reject_bool
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier216wpjit_build_list
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier217wpjit_list_repeat
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier213wpjit_call_py
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5tier211MethodEntryEBF_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215try_native_call
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier217enter_scalar_leaf
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier214wpjit_list_get
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons23clear_thread_python_tls
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtB7_6object6ObjectB1o_EEE21try_borrow_mut_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13builtin_types26clear_thread_type_registry
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecAmj40_E21reserve_one_uncheckedCs4N3LIUsVkSw_17cranelift_codegen
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10signal_mod25unregister_thread_pthread
+__RNvCskHi7NRqR3VY_10weavepy_vm13slice_indices
+__RNvNtCskHi7NRqR3VY_10weavepy_vm13vm_singletons22clear_worker_thread_id
+__RINvMs_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectE5drainINtNtNtCs2NHf1dqD6Ob_4core3ops5range14RangeInclusivejEEBJ_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCsl3bTtyQSXML_8lock_api5mutex5MutexNtNtCsfqmARTQr4sq_11parking_lot9raw_mutex8RawMutexINtNtCs2NHf1dqD6Ob_4core6option6OptionIBx_NtNtCskHi7NRqR3VY_10weavepy_vm15thread_registry11ThreadEntryEEEE9drop_slowB2S_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_ordINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecANtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst10blockorder12LoweredBlockj80_E21reserve_one_uncheckedBO_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_ord
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCskHi7NRqR3VY_10weavepy_vm5tier27JitBufsEEB1J_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecANtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst6buffer12MachCallSitej10_E21reserve_one_uncheckedBO_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecAINtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst6buffer10MachSrcLocNtBL_7StencilEj40_E21reserve_one_uncheckedBP_
+__RNvMsc_Csf2xUKSCCMOo_8smallvecINtB5_8SmallVecAINtNtNtCs4N3LIUsVkSw_17cranelift_codegen8machinst6buffer10MachSrcLocNtBL_5FinalEj40_E8try_growBP_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtCskHi7NRqR3VY_10weavepy_vm4sync7GilCellINtNtB7_3vec3VecNtNtBL_5error11PyExceptionEEE9drop_slowBL_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCINvMs18_NtCskHi7NRqR3VY_10weavepy_vm6objectNtBL_9BuiltinFn11with_kwargsNCNvNtNtBN_6stdlib11thread_real16make_lock_object0E0EBN_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real16make_lock_objects0_0EBJ_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtCskHi7NRqR3VY_10weavepy_vm4sync8RealLockE9drop_slowBK_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object13int_from_i128
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real25make_thread_handle_objects_0B7_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17do_list_sort_call
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object6ObjectE17try_borrow_lockedB7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object6ObjectE21try_borrow_mut_lockedB7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object12LazyIterKindE21try_borrow_mut_lockedB7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object12LazyIterKindE17try_borrow_lockedB7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCs2DfPjnIE6AW_8indexmap3set8IndexSetNtNtB7_6object7DictKeyNtNtB7_8fasthash13FxBuildHasherEE21try_borrow_mut_lockedB7_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCs4Ez510r8k3d_11weavepy_jit2ir5TFuncECskHi7NRqR3VY_10weavepy_vm
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel10make_dumps0B7_
+__RNvMs0_NtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder4save
+__RNvMs0_NtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder7memoize
+__RNvMs0_NtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder10save_items
+__RNvMNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel6encodeNtB2_6Framer4data
+__RNvMs0_NtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder7integer
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtB4_6option6OptionNtNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel6encode15InstanceContextEEB15_
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel10make_loads0B7_
+__RNvMs4_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accelNtB5_14UnpicklerGuard5holds
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14int_bit_lengthINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14int_bit_length
+__RNvMsn_NtCslSKxsPn8Dnr_10num_bigint6bigintNtB5_6BigInt3pow
+__RNvXsa_NtNtCslSKxsPn8Dnr_10num_bigint7biguint5powerNtB7_7BigUintINtNtCs39fLfHtJxK3_10num_traits3pow3PowmE3pow
+__RNvXs8_NtNtCslSKxsPn8Dnr_10num_bigint7biguint14multiplicationRNtB7_7BigUintNtNtNtCs2NHf1dqD6Ob_4core3ops5arith3Mul3mul
+__RNvNtNtCslSKxsPn8Dnr_10num_bigint7biguint14multiplication10scalar_mul
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VechE6resizeCskHi7NRqR3VY_10weavepy_vm
+__RNvXsa_NtNtCslSKxsPn8Dnr_10num_bigint7biguint14multiplicationNtB7_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith9MulAssignRBX_E10mul_assign
+__RNvYNvNvMNtCskHi7NRqR3VY_10weavepy_vm13builtin_typesNtB7_12BuiltinTypes5build9proxy_newINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB9_6object6ObjectEE4callB9_
+__RNvNvMNtCskHi7NRqR3VY_10weavepy_vm13builtin_typesNtB4_12BuiltinTypes5build9proxy_new
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_chrINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_chr
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object16range_len_bigint
+__RNvMss_NtCskHi7NRqR3VY_10weavepy_vm6objectNtB5_5Range6bounds
+__RNvXs0_NtNtCslSKxsPn8Dnr_10num_bigint6bigint11subtractionNtB7_6BigIntINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3SubRBT_E3sub
+__RNvMs1_NtCsfqmARTQr4sq_11parking_lot9raw_mutexNtB5_8RawMutex11unlock_slow
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real20join_wait_collecting
+__RNvCskHi7NRqR3VY_10weavepy_vm11range_slice
+__RNCNvCskHi7NRqR3VY_10weavepy_vm11range_slice0B3_
+__RNvXNtNtCslSKxsPn8Dnr_10num_bigint6bigint11subtractionRNtB4_6BigIntNtNtNtCs2NHf1dqD6Ob_4core3ops5arith3Sub3sub
+__RINvNtNtNtNtCs6KxzipSu1yP_3std3sys12thread_local6native4lazy7destroyNtNtCs8TNC6SO7etN_16parking_lot_core11parking_lot10ThreadDataECsfqmARTQr4sq_11parking_lot
+__RNCNvCskHi7NRqR3VY_10weavepy_vm11range_slices_0B3_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel4NodeEEB1e_
+__RNvXsf_NtNtCslSKxsPn8Dnr_10num_bigint6bigint14multiplicationRNtB7_6BigIntNtNtNtCs2NHf1dqD6Ob_4core3ops5arith3Mul3mul
+__RNvXs_NtNtCslSKxsPn8Dnr_10num_bigint6bigint8additionRNtB6_6BigIntINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3AddBP_E3add
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier217wpjit_call_method
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter15do_min_max_call
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter10op_compare
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter21deep_order_collection
+__RNvCskHi7NRqR3VY_10weavepy_vm12is_nan_value
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier218wpjit_dyn_attr_get
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7sre_mod11sre_compile
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7sre_mod14codeseq_to_vec0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7sre_mod8sre_exec
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib7sre_mod14subject_to_vec
+__RNvMs0_NtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel6encodeNtB5_7Encoder13save_instance
+__RNCNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel10make_dumps00B9_
+__RNvMs1_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accelNtB5_11ImportGuard7modules
+__RNvMs0_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accelNtB5_12BuiltinGuard5holds
+__RNvMs_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accelNtB4_20ModuleFunctionsGuard5holds
+__RNvNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel7classes12plain_layout
+__RNvNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel7classes13is_heap_class
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueINtNtCsdq5mISKDh0s_5alloc5boxed3BoxNtNtCskHi7NRqR3VY_10weavepy_vm5tier27CallCtxEEB1e_
+__RNvNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel7classes16benign_metaclass
+__RNvNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel7classes14resolve_global
+__RNvNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel7classes10slot_names
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier221finish_deopted_callee
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel5build
+__RNvNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel7classes14is_member_slot
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel1b
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel6encode9ClassPlanE9drop_slowBO_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier213rebuild_stack
+__RNvNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface19install_type_checks18type_instancecheck
+__RNCNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib12pickle_accel10make_loads00B9_
+__RINvMs2_NtNtCs6KxzipSu1yP_3std6thread5localINtB6_8LocalKeyINtNtCs2NHf1dqD6Ob_4core4cell7RefCellNtNtCskHi7NRqR3VY_10weavepy_vm5tier28JitStateEE4withNCNvB1x_16try_enter_resume0INtNtBZ_6option6OptionNtB1x_13CompiledEntryEEB1z_
+__RNvCskHi7NRqR3VY_10weavepy_vm15exception_value
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object10repr_enter
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object10repr_leave
+__RNvXs0_NtNtCslSKxsPn8Dnr_10num_bigint6bigint8additionNtB7_6BigIntINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3AddRBP_E3add
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5tier212NativeCalleeEBF_
+__RNvXsc_NtNtCslSKxsPn8Dnr_10num_bigint6bigint11subtractionNtB7_6BigIntINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3SublE3sub
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier218finish_interp_call
+__RNvXs2_NtNtCslSKxsPn8Dnr_10num_bigint7biguint11subtractionNtB7_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith9SubAssignmE10sub_assign
+__RNvNtNtCslSKxsPn8Dnr_10num_bigint7biguint11subtraction4sub2
+__RNvXsF_NtNtCslSKxsPn8Dnr_10num_bigint6bigint8divisionNtB7_6BigIntNtNtNtCs2NHf1dqD6Ob_4core3ops5arith3Div3div
+__RNvMs6_NtCsdq5mISKDh0s_5alloc2rcINtB5_2RcNtNtCskHi7NRqR3VY_10weavepy_vm5tier211DirectEntryE9drop_slowBH_
+__RNvNtNtCslSKxsPn8Dnr_10num_bigint7biguint8division11div_rem_ref
+__RNvMs1_NtCseIGkHRfguhS_9hashbrown3mapINtB5_7HashMapmTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeEENtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE6insertB1r_
+__RINvMs6_NtCseIGkHRfguhS_9hashbrown3rawINtB6_8RawTableTmTmINtNtCsdq5mISKDh0s_5alloc3vec3VecNtNtCs4Ez510r8k3d_11weavepy_jit5value7JitTypeEEEE14reserve_rehashNCINvNtB8_3map11make_hashermBR_NtNtNtCs6KxzipSu1yP_3std4hash6random11RandomStateE0EB1u_
+__RNCNvNvNtCskHi7NRqR3VY_10weavepy_vm12type_surface21install_value_richcmp15richcmp_builtin0B7_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins13dict_fromkeysINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins13dict_fromkeys
+__RNvXNtNtCslSKxsPn8Dnr_10num_bigint7biguint11subtractionNtB4_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith3SubRBR_E3sub
+__RNvXs_NtNtCslSKxsPn8Dnr_10num_bigint7biguint11subtractionNtB6_7BigUintINtNtNtCs2NHf1dqD6Ob_4core3ops5arith9SubAssignRBT_E10sub_assign
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11list_removeINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter17make_seq_iterator
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11list_remove
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object9member_eq
+__RINvNtNtNtCs2NHf1dqD6Ob_4core5slice4sort8unstable7ipnsortTjjNtNtCs7uvW3w2TaMW_16cranelift_module6module6FuncIdENCINvMB6_SBT_20sort_unstable_by_keyjNCNvMs0_NtCs3m1Fwy4cFPt_13cranelift_jit7backendNtB2t_9JITModule20finalize_definitionss0_0E0EB2v_
+__RNvNtCs4Ez510r8k3d_11weavepy_jit7analyze17unary_result_type
+__RNCNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15analyze_for_rets2_0B5_
+__RNCNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15analyze_for_rets1_0B5_
+__RNCNvNtCs4Ez510r8k3d_11weavepy_jit7analyze15analyze_for_rets8_0B5_
+__RNvCskHi7NRqR3VY_10weavepy_vm8unary_op
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier212wpjit_str_eq
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier213wpjit_str_get
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier215wpjit_const_str
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier218wpjit_dyn_attr_set
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtB7_6object6ObjectEE21try_borrow_mut_lockedB7_
+__RNvMs4_NtCsdq5mISKDh0s_5alloc7raw_vecINtB5_6RawVecTNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectBN_EE8grow_oneBR_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellmE21try_borrow_mut_lockedB7_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11set_discardINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11set_discard
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier221native_exit_writeback
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object11FileBackendE17try_borrow_lockedB7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCsdq5mISKDh0s_5alloc6string6StringEE17try_borrow_lockedB7_
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real25make_thread_handle_object0B7_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object10PyIteratorE17try_borrow_lockedB7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real19is_main_interpreter
+__RINvNtNtNtNtCs2NHf1dqD6Ob_4core5slice4sort6shared5pivot11median3_recjNvYjNtNtBa_3cmp10PartialOrd2ltECs4Ez510r8k3d_11weavepy_jit
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real25make_thread_handle_objects0_0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib11thread_real15thread_shutdown
+__RNvMs0_NtCskHi7NRqR3VY_10weavepy_vm15thread_registryNtB5_14ThreadRegistry15join_non_daemon
+__RNvNtCskHi7NRqR3VY_10weavepy_vm5tier216probe_ctor_field
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNtNtCskHi7NRqR3VY_10weavepy_vm5tier214ClassCtorEntryEBF_
+__RNvMs1_NtNtCs6KxzipSu1yP_3std6thread9lifecycleINtB5_9JoinInneruE4joinCskHi7NRqR3VY_10weavepy_vm
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtNtCs6KxzipSu1yP_3std6thread9lifecycle6PacketuEE9drop_slowCskHi7NRqR3VY_10weavepy_vm
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm8builtins21bytes_from_source_obj0B5_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins18bytearray_byte_arg
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10bytes_findINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtCs2NHf1dqD6Ob_4core6option6OptionTNtNtB7_6object6ObjectIBR_B20_EIBR_TNtNtBV_6string6StringB20_EEEEEE17try_borrow_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins10bytes_find
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins20bytes_needle_guarded
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecINtNtCs2NHf1dqD6Ob_4core6option6OptionTNtNtB7_6object6ObjectIBR_B20_EIBR_TNtNtBV_6string6StringB20_EEEEEE21try_borrow_mut_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object24bytearray_export_acquire
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecyEE17try_borrow_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm6object24bytearray_export_release
+__RNvNtCs3XEsNKy5wmG_6memchr6memmem4find
+__RINvMNtNtCs3XEsNKy5wmG_6memchr6memmem8searcherNtB3_8Searcher3newNtNtNtNtB7_4arch3all10packedpair20DefaultFrequencyRankECskHi7NRqR3VY_10weavepy_vm
+__RNvNtNtCs3XEsNKy5wmG_6memchr6memmem8searcher22searcher_kind_one_byte
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19dispatch_inplace_op
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellbE17try_borrow_lockedB7_
+__RNCNvNtCskHi7NRqR3VY_10weavepy_vm8builtins14numeric_dunders2_0B5_
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter13do_float_call
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VecTNtNtBV_6string6StringNtNtB7_6object6ObjectEEE17try_borrow_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins7b_float
+__RNCINvNtCskHi7NRqR3VY_10weavepy_vm8builtins9method_kwNvB4_18bytes_translate_kwEs_0B6_
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellINtNtCsdq5mISKDh0s_5alloc3vec3VechEE21try_borrow_mut_lockedB7_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins13bytes_argview
+__RNvMsf_NtCskHi7NRqR3VY_10weavepy_vm4syncINtB5_7GilCellNtNtB7_6object11FileBackendE21try_borrow_mut_lockedB7_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNvNtCskHi7NRqR3VY_10weavepy_vm8builtins18bytes_translate_kwE9drop_slowBK_
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11list_insertINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins11list_insert
+__RNvYNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_strINtNtNtCs2NHf1dqD6Ob_4core3ops8function2FnTRSNtNtB6_6object6ObjectEE4callB6_
+__RNvNtCskHi7NRqR3VY_10weavepy_vm8builtins5b_str
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter19format_template_str
+__RNvCskHi7NRqR3VY_10weavepy_vm17scan_format_field
+__RNvCskHi7NRqR3VY_10weavepy_vm18split_format_field
+__RNvCskHi7NRqR3VY_10weavepy_vm19split_name_trailers
+__RNvCskHi7NRqR3VY_10weavepy_vm14format_int_hex
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNvNtCskHi7NRqR3VY_10weavepy_vm8builtins13str_format_kwE9drop_slowBK_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel17json_make_scanner
+__RNvMs3_CskHi7NRqR3VY_10weavepy_vmNtB5_11Interpreter8op_truth
+__RNvCskHi7NRqR3VY_10weavepy_vm18long_overflows_f64
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel5builds_0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel17json_make_encoder
+__RNvMs1_NtCsdq5mISKDh0s_5alloc3vecINtB5_3VecINtNtCs2NHf1dqD6Ob_4core6option6OptionNtNtCskHi7NRqR3VY_10weavepy_vm6object6ObjectEE6resizeB1l_
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel17json_make_encoders5_0B7_
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel12encode_value
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel17encode_value_impl
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel9json_kind
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accelNtB5_10JsonWriter8push_str
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel20write_encoded_string
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accelNtB5_10JsonWriter11quoted_utf8
+__RNvMs3_NtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accelNtB5_10JsonWriter11push_object
+__RNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel11encode_list0B7_
+__RINvNtCs2NHf1dqD6Ob_4core3ptr9drop_glueNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel17json_make_encoders5_0EBJ_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcNtNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel10EncoderCfgE9drop_slowBM_
+__RNvMsn_NtCsdq5mISKDh0s_5alloc4syncINtB5_3ArcINtNtB7_3vec3VecmEE9drop_slowCskHi7NRqR3VY_10weavepy_vm
+__RNCNCNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel17json_make_scanners_00B9_
+__RINvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel9scan_oncehEB6_
+__RINvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel12parse_objecthEB6_
+__RINvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel21scan_string_with_memohEB6_
+__RNvXNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accelhNtB2_8JsonChar3key
+__RINvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel11parse_arrayhEB6_
+__RNvXNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accelhNtB2_8JsonChar6string
+__RNvNtNtCskHi7NRqR3VY_10weavepy_vm6stdlib10json_accel13finish_object

From 83f65b9fe6dbb104ae701906488845a14ec6eec2 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 14:16:52 -0700
Subject: [PATCH 076/173] perf: trim the leaf attribute cache's lookup and tier
 2's phase reads

Two costs the earlier memory changes added to hot paths:

- The leaf attribute cache's lookup built a slice of the table and
  bounds-checked both probes. It now takes the mask from the table
  address's tag and indexes within it unchecked. Under callgrind,
  oo_patterns' instance_dunder_function went from 547 back to 472
  instructions per iteration (435 before the cache shrank).
- Tier 2's counter bump read the compilation phase (a thread-local) on
  every call of cold code to place the code generator's warm-up. It now
  reads the phase only at the two counts where a warm-up can start.

oo_patterns: 71,393 -> 71,133 instructions per iteration (70,971 before
this branch's memory work).
---
 crates/weavepy-vm/src/tier2.rs | 28 ++++++++++++++++++++--------
 crates/weavepy-vm/src/types.rs | 22 +++++++++++++++++-----
 2 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index 20608e83..ff623fa3 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -1098,6 +1098,16 @@ fn compile_allowed(counter: u32, threshold: u32) -> bool {
 /// attempt costs milliseconds.
 const IMPORT_THRESHOLD_FACTOR: u32 = 64;
 
+/// The work at which the current phase admits compiling code whose
+/// threshold is `threshold` (see [`compile_allowed`]).
+fn admit_at(threshold: u32) -> u32 {
+    if compile_allowed(0, threshold) {
+        threshold
+    } else {
+        threshold.saturating_mul(IMPORT_THRESHOLD_FACTOR)
+    }
+}
+
 /// Bodies longer than this many instructions need proportionally more
 /// work before start-up or an import analyzes them (see
 /// [`import_size_budget`]).
@@ -1335,18 +1345,20 @@ impl JitState {
                         entry.calls = entry.calls.wrapping_add(1);
                     }
                     // Halfway to the work this phase admits a compile at
-                    // (start-up and imports need far more first).
-                    let admit_at = if compile_allowed(0, self.threshold) {
-                        self.threshold
-                    } else {
-                        self.threshold.saturating_mul(IMPORT_THRESHOLD_FACTOR)
-                    };
-                    if entry.counter == admit_at / 2 && self.engine.is_none() {
+                    // (start-up and imports need far more first). (The
+                    // phase is read only at the two counts it can be.)
+                    let half = self.threshold / 2;
+                    if (entry.counter == half
+                        || entry.counter
+                            == self.threshold.saturating_mul(IMPORT_THRESHOLD_FACTOR) / 2)
+                        && entry.counter == admit_at(self.threshold) / 2
+                        && self.engine.is_none()
+                    {
                         spawn_codegen_prewarm();
                     }
                     if entry.counter < self.threshold
                         || !compile_allowed(entry.counter, self.threshold)
-                        || entry.counter < import_size_budget(code, admit_at)
+                        || entry.counter < import_size_budget(code, admit_at(self.threshold))
                     {
                         return None;
                     }
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index e42278bc..f576a62f 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -573,7 +573,7 @@ impl LeafAttrCache {
     }
 
     /// The table, if one was allocated.
-    #[inline]
+    #[cfg(test)]
     fn table(&self) -> Option<&[LeafAttrEntry]> {
         let t = self.0.get();
         if t == 0 {
@@ -666,14 +666,26 @@ impl LeafAttrCache {
     /// The cached kind for `name` under `ver`, if present.
     #[inline]
     pub fn get(&self, name: usize, ver: u64) -> Option<&LeafAttrKind> {
-        let t = self.table()?;
+        let t = self.0.get();
+        if t == 0 {
+            return None;
+        }
         // (Both table lengths are powers of two.)
-        let mask = t.len() - 1;
-        let e = &t[Self::index(name) & mask];
+        let mask = if t & LEAF_ATTRS_SMALL_TAG != 0 {
+            LEAF_ATTRS_SMALL - 1
+        } else {
+            LEAF_ATTRS_LARGE - 1
+        };
+        let base = (t & !LEAF_ATTRS_SMALL_TAG) as *const LeafAttrEntry;
+        // SAFETY: a nonzero address (tag removed) is a live table of
+        // `mask + 1` entries, indexed below within the mask; GIL-serialized,
+        // and no `&mut` escapes `set`.
+        let e = unsafe { &*base.add(Self::index(name) & mask) };
         if e.name == name && e.ver == ver {
             return Some(&e.kind);
         }
-        let e = &t[Self::alt(name) & mask];
+        // SAFETY: as above.
+        let e = unsafe { &*base.add(Self::alt(name) & mask) };
         (e.name == name && e.ver == ver).then_some(&e.kind)
     }
 

From 665d035536245a65007044b38c84a4848ffd534e Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 14:54:38 -0700
Subject: [PATCH 077/173] perf: copy a Name's text without the formatting
 machinery

`to_string()` on a Name went through `Display`, so every function made
from a code object (`new_function` copies the code's name) formatted
its name with `Formatter::pad` and a growing `String`. Under callgrind
that added about 600 instructions per iteration of closures_decorators,
which makes functions per call.

Name now has an inherent `to_string` that copies the text, which every
caller picks up. closures_decorators: 84,756 -> 84,141 instructions per
iteration (83,892 before this branch's memory work).

Also hash eight-byte chunks with `as_chunks` and drop two conversions
clippy flagged as useless.
---
 crates/weavepy-compiler/src/lib.rs  |  4 ++--
 crates/weavepy-compiler/src/name.rs | 16 ++++++++++++----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs
index 48a1dac8..a2dc3920 100644
--- a/crates/weavepy-compiler/src/lib.rs
+++ b/crates/weavepy-compiler/src/lib.rs
@@ -7572,7 +7572,7 @@ impl Compiler {
             if matches!(self.bindings.get(free.as_str()), Some(Binding::Local)) {
                 self.bindings.insert(free.to_string(), Binding::Cell);
                 if !has_name(&self.co.cellvars, free) {
-                    self.co.cellvars.push(Name::from(free.clone()));
+                    self.co.cellvars.push(free.clone());
                 }
             }
         }
@@ -12089,7 +12089,7 @@ impl Compiler {
             if matches!(self.bindings.get(free.as_str()), Some(Binding::Local)) {
                 self.bindings.insert(free.to_string(), Binding::Cell);
                 if !has_name(&self.co.cellvars, free) {
-                    self.co.cellvars.push(Name::from(free.clone()));
+                    self.co.cellvars.push(free.clone());
                 }
             }
         }
diff --git a/crates/weavepy-compiler/src/name.rs b/crates/weavepy-compiler/src/name.rs
index 0ed7a973..71a19aa2 100644
--- a/crates/weavepy-compiler/src/name.rs
+++ b/crates/weavepy-compiler/src/name.rs
@@ -42,6 +42,15 @@ impl Name {
         &self.0
     }
 
+    /// The text as a `String`: a copy, without the formatting machinery
+    /// `ToString` goes through for a type that isn't `str` (a function
+    /// made per call takes its name this way).
+    #[allow(clippy::inherent_to_string_shadow_display)]
+    #[inline]
+    pub fn to_string(&self) -> String {
+        String::from(&*self.0)
+    }
+
     /// Whether `a` and `b` share one allocation (equal pooled names always
     /// do).
     #[inline]
@@ -68,11 +77,10 @@ impl Hasher for Fx {
     }
 
     fn write(&mut self, bytes: &[u8]) {
-        let mut chunks = bytes.chunks_exact(8);
-        for c in &mut chunks {
-            self.add(u64::from_le_bytes(c.try_into().expect("eight bytes")));
+        let (chunks, rest) = bytes.as_chunks::<8>();
+        for &c in chunks {
+            self.add(u64::from_le_bytes(c));
         }
-        let rest = chunks.remainder();
         if !rest.is_empty() {
             let mut last = [0u8; 8];
             last[..rest.len()].copy_from_slice(rest);

From 543014e1b2bed158d9af205e9e92366fa2a55a26 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 08:49:19 -0700
Subject: [PATCH 078/173] perf: read and write __slots__ members by their
 layout position

A slotted class's instances already lay their members out over names
the class shares, but every attribute site still found a member by
decoding its inline cache, re-borrowing the slot table and comparing
the member's name, about 130 instructions a read and 120 a store.

Attribute sites now remember a laid-out member's position in their
field shortcut, as CPython's member descriptors keep an offset: the
class version and the position, marked as a slot. Reads through the
core loop, the frameless leaf evaluator and tier 2 take the member at
that position once the instance's slots are laid out over its class's
layout, the frame JIT reads it in line, and stores write it in place.
A store-only `__init__` of a slotted class fills a fresh instance's
members directly, and a recycled instance keeps its unset member array
for the next tenant instead of reallocating it.

raytrace -15%, generators_tree -6%, mini_interpreter -4% instructions.
---
 crates/weavepy-vm/src/frame_jit.rs      | 100 ++++++---
 crates/weavepy-vm/src/lib.rs            | 197 +++++++++++++++--
 crates/weavepy-vm/src/types.rs          | 118 +++++++++-
 tests/regrtest/test_slot_field_sites.py | 272 ++++++++++++++++++++++++
 4 files changed, 636 insertions(+), 51 deletions(-)
 create mode 100644 tests/regrtest/test_slot_field_sites.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 09f72b33..f9bd8b00 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3504,6 +3504,11 @@ fn slot_note(code: &CodeObject, recv: &Object, pc: usize, cache: *const SlotCach
             let cache = unsafe { &*cache };
             cache.ver.set(ver);
             cache.idx.set(key_idx);
+            // (The compiled read takes a laid-out member in line.)
+            if let Some(ext) = crate::code_vm_ext_existing(code) {
+                let name = code.instructions[pc].arg;
+                crate::slot_field_note(code, ext, pc, inst, key_idx, name);
+            }
         }
     }
 }
@@ -9250,35 +9255,72 @@ impl<'a> Lower<'a> {
                     .iconst(ptr, std::ptr::from_ref(cache_slot).cast::<u8>() as i64);
                 let want = self.b.ins().load(types::I64, FLAGS, cache, FIELD_VER);
                 let idx = self.b.ins().uload32(FLAGS, cache, FIELD_IDX);
-                let lazy = self.b.ins().load(ptr, FLAGS, inst, l.inst_dict_lazy);
                 let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64);
                 let shared = self.b.ins().uload8(types::I64, FLAGS, flag, 0);
-                let borrow = self.b.ins().sload32(FLAGS, inst, l.inst_split_borrow);
-                let block = self.b.ins().load(ptr, FLAGS, inst, l.inst_split_block);
                 let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0);
                 let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want);
-                let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0);
-                let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0);
-                let other = self.b.ins().bor(lazy, shared);
-                let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0);
-                let bad = self.b.ins().bor(unset, stale);
-                let bad = self.b.ins().bor(bad, busy);
-                let bad = self.b.ins().bor(bad, empty);
-                let bad = self.b.ins().bor(bad, other);
-                self.branch_out(bad, miss);
-                let keys = self.b.ins().load(ptr, FLAGS, block, l.split_keys);
-                let ckeys = self.b.ins().load(ptr, FLAGS, cls, l.type_shared_keys);
-                let len = self.b.ins().uload32(FLAGS, block, l.split_len);
-                let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys);
-                let absent = self
-                    .b
-                    .ins()
-                    .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len);
-                let bad = self.b.ins().bor(foreign, absent);
-                self.branch_out(bad, miss);
-                let off = self.b.ins().ishl_imm(idx, 4);
-                let v = self.b.ins().iadd(block, off);
-                let v = self.b.ins().iadd_imm(v, i64::from(l.split_values));
+                let v = if cache_slot.get().1 & crate::SLOT_FIELD != 0 && l.slots_ok {
+                    // A `__slots__` member laid out over the class's own
+                    // layout, at the site's position, and set (see
+                    // `PyInstance::laid_out_slot`).
+                    let split = self.b.ins().band_imm(idx, i64::from(crate::SLOT_FIELD));
+                    let split = self.b.ins().icmp_imm(IntCC::Equal, split, 0);
+                    let shared = self.b.ins().icmp_imm(IntCC::NotEqual, shared, 0);
+                    let bad = self.b.ins().bor(unset, stale);
+                    let bad = self.b.ins().bor(bad, split);
+                    let bad = self.b.ins().bor(bad, shared);
+                    self.branch_out(bad, miss);
+                    let clayout = self.b.ins().load(ptr, FLAGS, cls, type_slot_layout(&l));
+                    let borrow = self.b.ins().sload32(FLAGS, inst, l.inst_slots_borrow);
+                    let form = self.b.ins().uload8(types::I32, FLAGS, inst, l.inst_slots_tag);
+                    let names = self.b.ins().load(ptr, FLAGS, inst, l.inst_slots_layout);
+                    let vals = self.b.ins().load(ptr, FLAGS, inst, l.inst_slots_values);
+                    let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0);
+                    let other =
+                        self.b
+                            .ins()
+                            .icmp_imm(IntCC::NotEqual, form, i64::from(l.slots_laid_out));
+                    let foreign = self.b.ins().icmp(IntCC::NotEqual, names, clayout);
+                    let bad = self.b.ins().bor(busy, other);
+                    let bad = self.b.ins().bor(bad, foreign);
+                    self.branch_out(bad, miss);
+                    let i = self.b.ins().band_imm(idx, i64::from(!crate::SLOT_FIELD));
+                    let off = self.b.ins().ishl_imm(i, 4);
+                    let v = self.b.ins().iadd(vals, off);
+                    let vt = self.b.ins().uload8(types::I32, FLAGS, v, 0);
+                    let none = self
+                        .b
+                        .ins()
+                        .icmp_imm(IntCC::Equal, vt, i64::from(l.tag_unbound));
+                    self.branch_out(none, miss);
+                    v
+                } else {
+                    let lazy = self.b.ins().load(ptr, FLAGS, inst, l.inst_dict_lazy);
+                    let borrow = self.b.ins().sload32(FLAGS, inst, l.inst_split_borrow);
+                    let block = self.b.ins().load(ptr, FLAGS, inst, l.inst_split_block);
+                    let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0);
+                    let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0);
+                    let other = self.b.ins().bor(lazy, shared);
+                    let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0);
+                    let bad = self.b.ins().bor(unset, stale);
+                    let bad = self.b.ins().bor(bad, busy);
+                    let bad = self.b.ins().bor(bad, empty);
+                    let bad = self.b.ins().bor(bad, other);
+                    self.branch_out(bad, miss);
+                    let keys = self.b.ins().load(ptr, FLAGS, block, l.split_keys);
+                    let ckeys = self.b.ins().load(ptr, FLAGS, cls, l.type_shared_keys);
+                    let len = self.b.ins().uload32(FLAGS, block, l.split_len);
+                    let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys);
+                    let absent = self
+                        .b
+                        .ins()
+                        .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len);
+                    let bad = self.b.ins().bor(foreign, absent);
+                    self.branch_out(bad, miss);
+                    let off = self.b.ins().ishl_imm(idx, 4);
+                    let v = self.b.ins().iadd(block, off);
+                    self.b.ins().iadd_imm(v, i64::from(l.split_values))
+                };
                 match item {
                     // The receiver is the local's: the value just takes a
                     // copy.
@@ -9997,5 +10039,13 @@ impl<'a> Lower<'a> {
 }
 
 /// Where a field cache's version and position sit.
+/// Where a class keeps [`crate::types::TypeObject::slot_layout_word`],
+/// from its pointer (measured as `layout`'s `type_attr_version` is).
+fn type_slot_layout(l: &weavepy_jit::ObjLayout) -> i32 {
+    use crate::types::TypeObject;
+    let base = l.type_attr_version as isize - std::mem::offset_of!(TypeObject, attr_version) as isize;
+    (base + std::mem::offset_of!(TypeObject, slot_layout_word) as isize) as i32
+}
+
 const FIELD_VER: i32 = std::mem::offset_of!((u64, u32), 0) as i32;
 const FIELD_IDX: i32 = std::mem::offset_of!((u64, u32), 1) as i32;
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index de2f2e0f..badeeb5b 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -17295,7 +17295,37 @@ impl Interpreter {
         {
             return false;
         }
-        if let Some(proving) = proving {
+        // A class that lays its `__slots__` members out: each store's site
+        // must have last stored a distinct member of this class version's
+        // instances (sites another version recorded say nothing), whose
+        // layout positions the stores then fill. A site that recorded
+        // anything else for this version never will.
+        let laid_out = ty.slot_layout.get().and_then(Option::as_ref);
+        let mut members = [0u8; 16];
+        if laid_out.is_some() {
+            let mut seen = 0u64;
+            for (k, &(pc, _)) in shape.stores.iter().enumerate() {
+                match slots.get(pc as usize).map(FieldSlot::get) {
+                    Some((v, idx))
+                        if v == ver
+                            && idx & SLOT_FIELD != 0
+                            && (idx & !SLOT_FIELD) < 64
+                            && seen & (1 << (idx & !SLOT_FIELD)) == 0 =>
+                    {
+                        let i = idx & !SLOT_FIELD;
+                        seen |= 1 << i;
+                        members[k] = i as u8;
+                    }
+                    Some((v, _)) if v == ver => {
+                        if let Some(proving) = proving {
+                            proving.store(2, std::sync::atomic::Ordering::Relaxed);
+                        }
+                        return false;
+                    }
+                    _ => return false,
+                }
+            }
+        } else if let Some(proving) = proving {
             // Every store's site must have last stored the next position
             // of this class version's instances. A site proven for another
             // position never will be; one that hasn't stored for this
@@ -17311,26 +17341,31 @@ impl Interpreter {
                 }
             }
         }
-        let Some(keys) = ty.shared_keys.get() else {
-            return false;
-        };
         let consts = code_vm_ext(code).map_or(&[][..], |e| &e.objects[..]);
-        if keys.len() < shape.stores.len()
-            || shape
-                .stores
-                .iter()
-                .any(|&(_, src)| matches!(src, StoreSrc::Const(c) if c as usize >= consts.len()))
+        if shape
+            .stores
+            .iter()
+            .any(|&(_, src)| matches!(src, StoreSrc::Const(c) if c as usize >= consts.len()))
         {
             return false;
         }
+        let keys = match laid_out {
+            Some(_) => None,
+            None => match ty.shared_keys.get() {
+                Some(keys) if keys.len() >= shape.stores.len() => Some(keys),
+                _ => return false,
+            },
+        };
         // Committed.
         let (inst, _) = self.alloc_plain_instance_obj(ty);
         let Object::Instance(i) = &inst else {
             unreachable!("a plain class's instance")
         };
-        // SAFETY: the instance is fresh and unshared.
-        if let Some(split) = unsafe { i.dict.split_cell().peek_mut() } {
-            split.reserve_for(|| ty.shared_keys.share(), keys.len());
+        if let Some(keys) = keys {
+            // SAFETY: the instance is fresh and unshared.
+            if let Some(split) = unsafe { i.dict.split_cell().peek_mut() } {
+                split.reserve_for(|| ty.shared_keys.share(), keys.len());
+            }
         }
         // The parameters: the instance, the arguments (moved out of the
         // stack on their last use), then the defaults.
@@ -17358,6 +17393,27 @@ impl Interpreter {
                 StoreSrc::Const(c) => consts[c as usize].clone(),
                 StoreSrc::Int(n) => Object::Int(n),
             };
+            if let Some(layout) = laid_out {
+                // The member's place (the write barrier first).
+                i.note_slot_store(&value);
+                // SAFETY: the instance is fresh and unshared; its slots
+                // were laid out over its class's layout at allocation.
+                let place = unsafe { i.slots.peek_mut() }
+                    .and_then(|s| s.values_for_layout_mut(layout))
+                    .and_then(|v| v.get_mut(usize::from(members[k])));
+                match place {
+                    Some(place) => drop(std::mem::replace(place, value)),
+                    None => {
+                        // (Unreachable for a fresh instance; the plain
+                        // store keeps it correct regardless.)
+                        let at = code.instructions[shape.stores[k].0 as usize].arg;
+                        if let Some(name) = code.names.get(at as usize) {
+                            i.slot_set(name, value);
+                        }
+                    }
+                }
+                continue;
+            }
             // SAFETY: a store with nothing else running; the positions are
             // the instance's next ones (proved above).
             if let Err(value) = unsafe { i.split_append(k, value) } {
@@ -19994,6 +20050,9 @@ impl Interpreter {
                 .get_index(key_idx as usize)
                 .filter(|(key, _)| slot_name_matches_in(names, code, name_idx, key))
                 .map(|(_, value)| value);
+            if indexed.is_some() {
+                slot_field_note(code, ext, cache_pc as usize, inst, key_idx, name_idx);
+            }
             // Slot order can vary by instance or after deletion.
             let value = indexed.or_else(|| slots.get(code.names.get(name_idx as usize)?))?;
             #[cfg(test)]
@@ -22216,6 +22275,9 @@ impl Interpreter {
                         .get_index(key_idx as usize)
                         .filter(|(key, _)| slot_name_matches(code, ins.arg, key))
                         .map(|(_, value)| value);
+                    if let (Some(_), Some(ext)) = (indexed, ext) {
+                        slot_field_note(code, ext, pc, inst, key_idx, ins.arg);
+                    }
                     indexed.or_else(|| slots.get(name))
                 }
                 _ => {
@@ -22297,7 +22359,21 @@ impl Interpreter {
         let cls = inst.cls_raw();
         let ext = code_vm_ext(code);
         if let Some((ver, idx)) = ext.and_then(|e| e.field_slot(attr_pc)).map(FieldSlot::get) {
-            if cls.attr_version.get() == ver && !crate::capi_watchers::dicts_active() {
+            if idx & SLOT_FIELD != 0 && cls.attr_version.get() == ver {
+                // A laid-out `__slots__` member, set or not: the member
+                // descriptor's store, in place.
+                // SAFETY: a store between two instructions (see `peek_mut`).
+                if let Some(slot) = unsafe {
+                    inst.laid_out_slot_mut((idx & !SLOT_FIELD) as usize, value.is_gc_atomic())
+                } {
+                    if !Self::core_droppable(slot) {
+                        return false;
+                    }
+                    // SAFETY: as the indexed store below.
+                    drop(std::mem::replace(slot, unsafe { std::ptr::read(value) }));
+                    return true;
+                }
+            } else if cls.attr_version.get() == ver && !crate::capi_watchers::dicts_active() {
                 // SAFETY: a store between two instructions (see `peek_mut`).
                 if let Some(slot) =
                     unsafe { inst.split_field_mut(idx as usize, value.is_gc_atomic()) }
@@ -22351,7 +22427,7 @@ impl Interpreter {
                 Self::core_store_new_attr(code, inst, cls, attr_pc, name_idx, ver, value)
             }
             IC::StoreAttrSlot { key_idx, ver } if cls.attr_version.get() == ver => {
-                Self::core_store_slot(code, inst, key_idx, name_idx, value)
+                Self::core_store_slot(code, inst, attr_pc, key_idx, name_idx, value)
             }
             _ => false,
         }
@@ -22366,6 +22442,7 @@ impl Interpreter {
     fn core_store_slot(
         code: &CodeObject,
         inst: &PyInstance,
+        attr_pc: usize,
         key_idx: u32,
         name_idx: u32,
         value: &Object,
@@ -22385,6 +22462,10 @@ impl Interpreter {
             let v = unsafe { std::ptr::read(value) };
             inst.note_slot_store(&v);
             drop(std::mem::replace(slot, v));
+            drop(slots);
+            if let Some(ext) = code_vm_ext_existing(code) {
+                slot_field_note(code, ext, attr_pc, inst, key_idx, name_idx);
+            }
             return true;
         }
         match slots.get_index_mut(key_idx as usize) {
@@ -22453,7 +22534,16 @@ impl Interpreter {
         cursor: &mut Option<usize>,
     ) -> Option<bool> {
         let (ver, idx) = ext.field_slot(attr_pc)?.get();
-        if inst.cls_raw().attr_version.get() != ver || crate::capi_watchers::dicts_active() {
+        if inst.cls_raw().attr_version.get() != ver {
+            return None;
+        }
+        if idx & SLOT_FIELD != 0 {
+            // A laid-out member's store, set or not (see `core_store_attr`).
+            // SAFETY: a read with nothing running (the commit's check pass).
+            let slot = unsafe { inst.laid_out_slot_any((idx & !SLOT_FIELD) as usize) }?;
+            return Some(Self::core_droppable(slot));
+        }
+        if crate::capi_watchers::dicts_active() {
             return None;
         }
         // SAFETY: a read with nothing running (the commit's check pass).
@@ -26661,7 +26751,21 @@ impl Interpreter {
                     .filter(|(key, _)| slot_name_matches(code, name_idx, key))
                     .map(|(_, value)| value);
                 match indexed {
-                    Some(v) => Some(Self::clone_operand(v)),
+                    Some(v) => {
+                        let v = Self::clone_operand(v);
+                        drop(slots);
+                        if let Some(ext) = code_vm_ext_existing(code) {
+                            slot_field_note(
+                                code,
+                                ext,
+                                cache_pc as usize,
+                                inst,
+                                key_idx,
+                                name_idx,
+                            );
+                        }
+                        Some(v)
+                    }
                     None => slots
                         .get(code.names.get(name_idx as usize)?.as_str())
                         .map(Self::clone_operand),
@@ -72420,7 +72524,10 @@ impl StampSlot {
 /// (see [`field_slot_note`]). The names are append-only and the version
 /// is process-unique, so while both match, the value at that position of
 /// a split instance *is* the attribute: no inline-cache decode and no
-/// name comparison.
+/// name comparison. A position marked [`SLOT_FIELD`] is instead a
+/// `__slots__` member's in its class's slot layout (see
+/// [`slot_field_note`]), which an instance laid out over that layout
+/// keeps there.
 #[repr(transparent)]
 struct FieldSlot(std::cell::UnsafeCell<(u64, u32)>);
 
@@ -72678,7 +72785,8 @@ enum SiteCall {
 }
 
 /// The attribute `inst` holds for the `LOAD_ATTR` at `pc`, through the
-/// site's [`FieldSlot`] (never recorded for a slot or native class).
+/// site's [`FieldSlot`]: a split field, or a laid-out `__slots__` member
+/// (never recorded for a native class).
 ///
 /// # Safety
 ///
@@ -72694,10 +72802,63 @@ unsafe fn field_slot_hit<'a>(
     if inst.cls_raw().attr_version.get() != ver {
         return None;
     }
+    if idx & SLOT_FIELD != 0 {
+        // SAFETY: forwarded contract.
+        return unsafe { inst.laid_out_slot((idx & !SLOT_FIELD) as usize) };
+    }
     // SAFETY: forwarded contract.
     unsafe { inst.split_field(idx as usize) }
 }
 
+/// A [`FieldSlot`] position naming a `__slots__` member's place in its
+/// class's slot layout (see [`crate::types::PyInstance::laid_out_slot`])
+/// rather than a split field's: CPython's member descriptor offset. No
+/// split position reaches it (a class shares at most
+/// [`crate::inst_dict::SHARED_KEYS_CAP`] names).
+pub(crate) const SLOT_FIELD: u32 = 1 << 31;
+
+/// Record the [`FieldSlot`] shortcut for the attribute site at `pc` of
+/// `code` (name `co_names[name_idx]`) to the `__slots__` member at
+/// position `idx` of `inst`'s slots, after the site's guarded read or
+/// store of a member (the class passed the site's version check): kept
+/// only when the instance's slots are laid out over its class's own
+/// layout with the name there, where the position names the member for
+/// as long as the class lives.
+#[inline(never)]
+fn slot_field_note(
+    code: &CodeObject,
+    ext: &CodeConstObjects,
+    pc: usize,
+    inst: &PyInstance,
+    idx: u32,
+    name_idx: u32,
+) {
+    let ninstrs = code.instructions.len();
+    let cls = inst.cls_raw();
+    let Some(layout) = cls.slot_layout.get().and_then(Option::as_ref) else {
+        return;
+    };
+    match layout.get(idx as usize) {
+        Some(key) if slot_name_matches_in(&ext.name_objs, code, name_idx, key) => {}
+        _ => return,
+    }
+    // SAFETY: a read with nothing running (the caller's own guarded
+    // access, whose borrow has ended).
+    if unsafe { inst.slots.peek() }
+        .and_then(|s| s.values_for_layout(layout))
+        .is_none()
+    {
+        return;
+    }
+    if !ext.field_slots.is_allocated() && !site_tables_warm(ext, ninstrs) {
+        return;
+    }
+    let slots = ext.alloc_sites(code, &ext.field_slots, FieldSlot::empty);
+    if let Some(slot) = slots.get(pc) {
+        slot.set((cls.attr_version.get(), idx | SLOT_FIELD));
+    }
+}
+
 /// Record the [`FieldSlot`] shortcut for the `LOAD_ATTR` at `pc`, after
 /// a guarded read found its name at dictionary position `idx` of `inst`
 /// (whose class passed the site's version check): kept only when that
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index f576a62f..1e023802 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -941,6 +941,10 @@ pub struct TypeObject {
     /// slots out over (see [`TypeObject::fresh_slots`]), decided at the
     /// first instance: `None` for a class that doesn't qualify.
     pub slot_layout: std::sync::OnceLock<Option<SharedSlice<DictKey>>>,
+    /// [`Self::slot_layout`]'s word (see [`SharedSlice::word`]) once it is
+    /// decided and present, `0` otherwise: compiled code compares an
+    /// instance's laid-out slots' names against it in line.
+    pub slot_layout_word: std::sync::atomic::AtomicUsize,
     /// Cached "do instances of this type carry a `__del__` finalizer
     /// anywhere in their MRO?" answer, so [`crate::object::PyInstance`]'s
     /// `Drop` safety net can skip an MRO walk on the hot per-instance drop
@@ -1375,6 +1379,7 @@ impl TypeObject {
             native_ext: std::sync::OnceLock::new(),
             abc_state: std::sync::OnceLock::new(),
             slot_layout: std::sync::OnceLock::new(),
+            slot_layout_word: std::sync::atomic::AtomicUsize::new(0),
             slot_names: RefCell::new(Vec::new()),
             declares_slots: Cell::new(false),
             forbids_dict: false,
@@ -1464,23 +1469,43 @@ impl TypeObject {
     /// in place, with no per-instance names and no growth.
     #[inline]
     pub fn fresh_slots(&self) -> SlotStorage {
-        match self.slot_layout.get_or_init(|| self.member_slot_layout()) {
+        match self.slot_layout_init() {
             Some(layout) => SlotStorage::unset_over(layout.clone()),
             None => SlotStorage::default(),
         }
     }
 
     /// Make `slots` this class's [`Self::fresh_slots`], leaving storage
-    /// that is already the empty default alone.
+    /// that is already the empty default alone, and reusing a retired
+    /// instance's unset laid-out values (see [`SlotStorage::retire`]).
     #[inline]
     pub(crate) fn reset_slots(&self, slots: &mut SlotStorage) {
-        match self.slot_layout.get_or_init(|| self.member_slot_layout()) {
-            Some(layout) => *slots = SlotStorage::unset_over(layout.clone()),
+        match self.slot_layout_init() {
+            Some(layout) => {
+                if !slots.reuse_unset_over(layout) {
+                    *slots = SlotStorage::unset_over(layout.clone());
+                }
+            }
             None if slots.is_empty_default() => {}
             None => *slots = SlotStorage::default(),
         }
     }
 
+    /// [`Self::slot_layout`], decided now if it isn't yet.
+    #[inline]
+    fn slot_layout_init(&self) -> &Option<SharedSlice<DictKey>> {
+        self.slot_layout.get_or_init(|| {
+            let layout = self.member_slot_layout();
+            if let Some(layout) = &layout {
+                self.slot_layout_word.store(
+                    SharedSlice::word(layout),
+                    std::sync::atomic::Ordering::Relaxed,
+                );
+            }
+            layout
+        })
+    }
+
     /// Whether the class is one whose native method bodies read its slots
     /// by the position its `__init__` sets them in (`collections.deque`,
     /// `contextvars.Context`, asyncio's handles and futures): those keep
@@ -2573,7 +2598,7 @@ impl SlotStorage {
 
     /// Access native fields only when the shared layout is the same
     /// allocation. Equal user-defined slot names aren't sufficient.
-    #[inline]
+    #[inline(always)]
     pub(crate) fn values_for_layout(&self, expected: &SharedSlice<DictKey>) -> Option<&[Object]> {
         match &self.data {
             SlotData::Fixed { layout, values } if SharedSlice::ptr_eq(layout, expected) => {
@@ -2620,6 +2645,38 @@ impl SlotStorage {
         )
     }
 
+    /// Empty the storage of a retiring instance whose values are all
+    /// atomic (releasing them runs no code): laid-out values are unset in
+    /// place, keeping their allocation for the next tenant (see
+    /// [`Self::reuse_unset_over`]); any other form becomes the default.
+    pub(crate) fn retire(&mut self) {
+        if let SlotData::Fixed { values, .. } = &mut self.data {
+            for v in values.iter_mut() {
+                *v = Object::Unbound;
+            }
+        } else if !self.is_empty_default() {
+            *self = Self::default();
+        }
+    }
+
+    /// Lay a retired storage (see [`Self::retire`]) out over `layout`, all
+    /// unset, reusing its values' allocation: `false` when it has none of
+    /// that size (the caller builds fresh storage).
+    #[inline]
+    pub(crate) fn reuse_unset_over(&mut self, layout: &SharedSlice<DictKey>) -> bool {
+        let SlotData::Fixed { layout: own, values } = &mut self.data else {
+            return false;
+        };
+        if values.len() != layout.len() {
+            return false;
+        }
+        if !SharedSlice::ptr_eq(own, layout) {
+            *own = layout.clone();
+        }
+        debug_assert!(values.iter().all(|v| matches!(v, Object::Unbound)));
+        true
+    }
+
     /// Storage holding a natively packed value (see [`SlotData::Packed`]).
     pub(crate) fn from_packed(packed: crate::stdlib::datetime_native::PackedSlots) -> Self {
         Self {
@@ -2714,6 +2771,17 @@ impl SlotStorage {
         false
     }
 
+    /// Empty a retiring instance's storage (see the 64-bit variant).
+    pub(crate) fn retire(&mut self) {
+        *self = Self::default();
+    }
+
+    /// Never laid out here (see the 64-bit variant).
+    #[inline]
+    pub(crate) fn reuse_unset_over(&mut self, _layout: &SharedSlice<DictKey>) -> bool {
+        false
+    }
+
     /// [`Self::index_of`] (never laid out here; see the 64-bit variant).
     pub fn store_position(&self, name: &str) -> Option<u32> {
         self.index_of(name)
@@ -3616,9 +3684,7 @@ impl PyInstance {
             d.map_mut_atomic_store().clear();
             d.reset_deferred_owner(owner);
         }
-        if !m.slots.get_mut().is_empty_default() {
-            *m.slots.get_mut() = SlotStorage::default();
-        }
+        m.slots.get_mut().retire();
         m.inline_values.set(true);
         m.deferred.set(true);
         m.hash_cache = crate::sync::CachedHash::new(None);
@@ -3752,6 +3818,42 @@ impl PyInstance {
         (!matches!(v, Object::Unbound)).then_some(v)
     }
 
+    /// [`Self::laid_out_slot`], set or not: what a store to the member
+    /// would replace.
+    ///
+    /// # Safety
+    ///
+    /// As [`crate::sync::GilCell::peek`].
+    #[inline(always)]
+    pub(crate) unsafe fn laid_out_slot_any(&self, i: usize) -> Option<&Object> {
+        let layout = self.cls_raw().slot_layout.get()?.as_ref()?;
+        // SAFETY: forwarded contract.
+        unsafe { self.slots.peek() }?
+            .values_for_layout(layout)?
+            .get(i)
+    }
+
+    /// [`Self::laid_out_slot`] for a store of a value of atomicity
+    /// `atomic`, set or not (a store fills an unset member): the write
+    /// barrier runs first (a non-atomic value starts tracking a deferred
+    /// instance), then the member's place.
+    ///
+    /// # Safety
+    ///
+    /// As [`crate::sync::GilCell::peek_mut`].
+    #[inline(always)]
+    #[allow(clippy::mut_from_ref)]
+    pub(crate) unsafe fn laid_out_slot_mut(&self, i: usize, atomic: bool) -> Option<&mut Object> {
+        let layout = self.cls_raw().slot_layout.get()?.as_ref()?;
+        if !atomic && self.deferred.get() {
+            self.ensure_gc_tracked();
+        }
+        // SAFETY: forwarded contract.
+        unsafe { self.slots.peek_mut() }?
+            .values_for_layout_mut(layout)?
+            .get_mut(i)
+    }
+
     /// Where [`Self::laid_out_slot`] finds member slot `name` on this
     /// instance: its position in the class's slot layout, when the
     /// instance's slots are laid out over it (set or not), and the
diff --git a/tests/regrtest/test_slot_field_sites.py b/tests/regrtest/test_slot_field_sites.py
new file mode 100644
index 00000000..ddce7133
--- /dev/null
+++ b/tests/regrtest/test_slot_field_sites.py
@@ -0,0 +1,272 @@
+"""Attribute sites that remember a `__slots__` member's layout position.
+
+A warm `x.attr` or `x.attr = v` site on a slotted class reads or writes
+the member by its position in the class's slot layout, and a store-only
+`__init__` fills a fresh instance's members directly. Each shortcut must
+step aside whenever the position stops naming the member: another class
+at the site, `__class__` assignment between layouts that order the same
+names differently, a class attribute replacing the member descriptor, an
+unset (or deleted) member, `__getattr__`/`__getattribute__` overrides,
+and a recycled instance taken over by another class. Releases must stay
+prompt, so `__del__` runs exactly when a slot's old value goes.
+"""
+
+import gc
+import unittest
+
+
+class XY:
+    __slots__ = ("x", "y")
+
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+
+
+class YX:
+    __slots__ = ("y", "x")
+
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+
+
+class XYZ(XY):
+    __slots__ = ("z",)
+
+    def __init__(self, x, y, z=0):
+        self.x = x
+        self.y = y
+        self.z = z
+
+
+class WithDict:
+    __slots__ = ("a", "__dict__")
+
+    def __init__(self, a, b):
+        self.a = a
+        self.b = b
+
+
+class Fallback:
+    __slots__ = ("x",)
+
+    def __getattr__(self, name):
+        return ("fallback", name)
+
+
+class Tracked:
+    log = []
+
+    def __init__(self, tag):
+        self.tag = tag
+
+    def __del__(self):
+        Tracked.log.append(self.tag)
+
+
+def read_xy(p):
+    return p.x * 10 + p.y
+
+
+def write_xy(p, x, y):
+    p.x = x
+    p.y = y
+
+
+class SlotFieldSiteTest(unittest.TestCase):
+    def test_reads_and_writes_warm(self):
+        p = XY(1, 2)
+        for i in range(3000):
+            write_xy(p, i, i + 1)
+            self.assertEqual(read_xy(p), i * 10 + i + 1)
+
+    def test_polymorphic_site(self):
+        objs = [XY(1, 2), YX(3, 4), XYZ(5, 6, 7)]
+        for _ in range(2000):
+            got = [read_xy(o) for o in objs]
+            self.assertEqual(got, [12, 34, 56])
+            for o in objs:
+                write_xy(o, o.y, o.x)
+                write_xy(o, o.y, o.x)
+
+    def test_class_assignment_between_orders(self):
+        for i in range(1500):
+            p = XY(i, -i)
+            self.assertEqual(read_xy(p), i * 10 - i)
+            p.__class__ = YX
+            self.assertEqual((p.x, p.y), (i, -i))
+            self.assertEqual(read_xy(p), i * 10 - i)
+            write_xy(p, -i, i)
+            self.assertEqual((p.x, p.y), (-i, i))
+            p.__class__ = XY
+            self.assertEqual(read_xy(p), -i * 10 + i)
+
+    def test_unset_and_deleted_members(self):
+        p = XY(1, 2)
+        for i in range(2000):
+            self.assertEqual(read_xy(p), 12)
+        del p.y
+        with self.assertRaises(AttributeError):
+            read_xy(p)
+        q = XY.__new__(XY)
+        with self.assertRaises(AttributeError):
+            read_xy(q)
+        q.y = 5
+        with self.assertRaises(AttributeError):
+            read_xy(q)
+        q.x = 4
+        self.assertEqual(read_xy(q), 45)
+
+    def test_getattr_fallback_on_unset_member(self):
+        f = Fallback()
+
+        def get(o):
+            return o.x
+
+        for _ in range(2000):
+            self.assertEqual(get(f), ("fallback", "x"))
+        f.x = 1
+        for _ in range(2000):
+            self.assertEqual(get(f), 1)
+        del f.x
+        self.assertEqual(get(f), ("fallback", "x"))
+
+    def test_member_replaced_by_class_attribute(self):
+        class C:
+            __slots__ = ("v",)
+
+            def __init__(self, v):
+                self.v = v
+
+        def get(o):
+            return o.v
+
+        c = C(3)
+        for _ in range(3000):
+            self.assertEqual(get(c), 3)
+        member = C.__dict__["v"]
+        C.v = property(lambda self: "prop")
+        self.assertEqual(get(c), "prop")
+        C.v = member
+        self.assertEqual(get(c), 3)
+
+    def test_getattribute_override_in_subclass(self):
+        class Loud(XY):
+            __slots__ = ()
+
+            def __getattribute__(self, name):
+                if name == "x":
+                    return 100
+                return object.__getattribute__(self, name)
+
+        p = XY(1, 2)
+        for _ in range(2000):
+            self.assertEqual(read_xy(p), 12)
+        self.assertEqual(read_xy(Loud(1, 2)), 1002)
+
+    def test_setattr_override(self):
+        class Upper(XY):
+            __slots__ = ()
+
+            def __setattr__(self, name, value):
+                object.__setattr__(self, name, value * 2)
+
+        p = XY(0, 0)
+        for i in range(2000):
+            write_xy(p, i, i)
+        u = Upper(1, 2)
+        self.assertEqual((u.x, u.y), (2, 4))
+        write_xy(u, 3, 4)
+        self.assertEqual((u.x, u.y), (6, 8))
+        object.__setattr__(u, "x", 7)
+        self.assertEqual(u.x, 7)
+
+    def test_store_only_init_shared_with_subclass(self):
+        class Base:
+            __slots__ = ("a", "b")
+
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+
+        class Pre(Base):
+            __slots__ = ("c",)
+
+        class Plain(Base):
+            pass
+
+        for i in range(2000):
+            b = Base(i, i + 1)
+            p = Pre(i, i + 2)
+            q = Plain(i, i + 3)
+            self.assertEqual((b.a, b.b), (i, i + 1))
+            self.assertEqual((p.a, p.b), (i, i + 2))
+            self.assertEqual((q.a, q.b), (i, i + 3))
+            self.assertFalse(hasattr(p, "c"))
+            self.assertEqual(q.__dict__, {})
+        w = WithDict(1, 2)
+        for i in range(2000):
+            w = WithDict(i, i * 2)
+        self.assertEqual((w.a, w.b, w.__dict__), (1999, 3998, {"b": 3998}))
+
+    def test_repeated_store_in_init(self):
+        class Twice:
+            __slots__ = ("v", "w")
+
+            def __init__(self, a, b):
+                self.v = a
+                self.v = b
+
+        for i in range(2000):
+            t = Twice(i, -i)
+            self.assertEqual(t.v, -i)
+            self.assertFalse(hasattr(t, "w"))
+
+    def test_recycled_instances_switch_classes(self):
+        for i in range(3000):
+            a = XY(i, 1)
+            b = YX(2, i)
+            c = XYZ(i, i, i)
+            self.assertEqual((a.x, a.y, b.x, b.y), (i, 1, 2, i))
+            self.assertEqual((c.x, c.y, c.z), (i, i, i))
+            del a, b, c
+            fresh = YX.__new__(YX)
+            self.assertFalse(hasattr(fresh, "x"))
+            self.assertFalse(hasattr(fresh, "y"))
+
+    def test_release_timing(self):
+        Tracked.log.clear()
+        p = XY(None, None)
+        for i in range(500):
+            write_xy(p, Tracked(i), None)
+            if i:
+                self.assertEqual(Tracked.log[-1], i - 1)
+        p.x = None
+        self.assertEqual(Tracked.log[-1], 499)
+        Tracked.log.clear()
+        for i in range(500):
+            q = XY(Tracked(i), 0)
+            del q
+            self.assertEqual(Tracked.log, [i])
+            Tracked.log.clear()
+
+    def test_self_reference_is_collected(self):
+        class Node:
+            __slots__ = ("me", "tag", "__weakref__")
+
+            def __init__(self, tag):
+                self.me = self
+                self.tag = tag
+
+        import weakref
+
+        refs = []
+        for i in range(500):
+            refs.append(weakref.ref(Node(i)))
+        gc.collect()
+        self.assertTrue(all(r() is None for r in refs))
+
+
+if __name__ == "__main__":
+    unittest.main()

From 21788bf5df53a4cbe3944379636e3464c5e01893 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 09:28:04 -0700
Subject: [PATCH 079/173] perf: find a lock's context-manager methods by their
 dict position

`with lock:` reached the lock's `__enter__`/`__exit__` through type-level
trampolines that hashed the method name and probed the lock's instance
dict on every call, about 200 instructions a call. Each lock's dict is
built in the same order, so a trampoline now remembers where it last
found its entry and settles the next lookup with one key comparison,
falling back to the hashed probe when the entry has moved.

with Lock/RLock: about -20% instructions per `with` block.
---
 crates/weavepy-vm/src/stdlib/thread_real.rs | 43 ++++++++++++++-------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs
index f596363b..40c28656 100644
--- a/crates/weavepy-vm/src/stdlib/thread_real.rs
+++ b/crates/weavepy-vm/src/stdlib/thread_real.rs
@@ -363,27 +363,44 @@ const TIMEOUT_MAX_SECS: f64 = 9_223_372_036.0;
 /// forwards to that instance's own closure, so both the VM's instance-dict
 /// path and a C extension's type-level lookup reach the identical
 /// implementation.
-/// The instance-dict entry `name` of `inst`, probed with `name`'s hash as
-/// `hash` caches it (computed on first use; `0` is "not yet").
+/// Where a trampoline last found its instance-dict entry: the name's hash
+/// (computed on first use; `0` is "not yet") and the entry's position.
+/// Every lock's dict is built in the same order, so the position settles a
+/// lookup with one key comparison.
+#[derive(Default)]
+struct ClosureSite {
+    hash: std::sync::atomic::AtomicI64,
+    at: std::sync::atomic::AtomicUsize,
+}
+
+/// The instance-dict entry `name` of `inst`, at the position `site` last
+/// found it in, else probed with `name`'s hash.
 fn instance_closure(
     inst: &crate::types::PyInstance,
     name: &'static str,
-    hash: &std::sync::atomic::AtomicI64,
+    site: &ClosureSite,
 ) -> Option<Object> {
     use std::sync::atomic::Ordering::Relaxed;
-    let mut h = hash.load(Relaxed);
+    let d = inst.dict_cell().borrow();
+    if let Some((DictKey(Object::Str(k)), v)) = d.get_index(site.at.load(Relaxed)) {
+        let k: &str = k;
+        // (The dict's keys are the same static names.)
+        if std::ptr::eq(k.as_ptr(), name.as_ptr()) && k.len() == name.len() || k == name {
+            return Some(v.clone());
+        }
+    }
+    let mut h = site.hash.load(Relaxed);
     if h == 0 {
         h = crate::object::py_str_hash(name);
-        hash.store(h, Relaxed);
+        site.hash.store(h, Relaxed);
     }
-    inst.dict_cell()
-        .borrow()
-        .get(&crate::object::StrKeyHashed { s: name, hash: h })
-        .cloned()
+    let (i, _, v) = d.get_full(&crate::object::StrKeyHashed { s: name, hash: h })?;
+    site.at.store(i, Relaxed);
+    Some(v.clone())
 }
 
 fn instance_dunder_trampoline(name: &'static str) -> Object {
-    let hash = std::sync::atomic::AtomicI64::new(0);
+    let site = ClosureSite::default();
     Object::Builtin(Rc::new(BuiltinFn {
         name,
         binds_instance: true,
@@ -391,7 +408,7 @@ fn instance_dunder_trampoline(name: &'static str) -> Object {
             let Some(Object::Instance(inst)) = args.first() else {
                 return Err(type_error(format!("{name}() requires an instance")));
             };
-            let closure = instance_closure(inst, name, &hash);
+            let closure = instance_closure(inst, name, &site);
             match closure {
                 Some(Object::Builtin(b)) => (b.call)(&args[1..]),
                 _ => Err(type_error(format!("instance has no {name}"))),
@@ -409,7 +426,7 @@ fn instance_dunder_trampoline(name: &'static str) -> Object {
 /// reachable via the type, not just the instance dict (RFC 0072 WS2).
 fn instance_method_trampoline(name: &'static str) -> Object {
     // (Shared by both halves below.)
-    let hash = std::sync::Arc::new(std::sync::atomic::AtomicI64::new(0));
+    let site = std::sync::Arc::new(ClosureSite::default());
     let forward =
         move |args: &[Object], kwargs: &[(String, Object)]| -> Result<Object, RuntimeError> {
             let Some(Object::Instance(inst)) = args.first() else {
@@ -417,7 +434,7 @@ fn instance_method_trampoline(name: &'static str) -> Object {
                     "descriptor '{name}' requires a lock instance"
                 )));
             };
-            let closure = instance_closure(inst, name, &hash);
+            let closure = instance_closure(inst, name, &site);
             match closure {
                 Some(Object::Builtin(b)) => match &b.call_kw {
                     Some(kw) => kw(&args[1..], kwargs),

From 342831c9fb187ae70e9715baa39bdae88f599b9b Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 09:28:05 -0700
Subject: [PATCH 080/173] perf: trim the builtin call lane and probe hasattr
 without binding

The core loop's builtin call lane cloned the builtin and its receiver
before moving the operands off the stack, and released the callable and
self slot through an array iterator. It now moves the receiver out of
the self slot, borrows the builtin from the callable it pops, and
releases each directly, keeping the full handler's release order.

`hasattr` ran `getattr`'s leaf half on cloned arguments and bound any
method it found only to drop it. The leaf half now works on borrowed
parts, and in its probe mode answers a class's function or a built-in
kind's method (`hasattr(stream, "write")`) as present without binding
it; anything it can't settle still takes the full call, so a getter's
non-AttributeError still propagates.

hasattr(StringIO, "write"): 3119 -> 844 instructions (CPython 745);
hasattr on an instance method: 1390 -> 972.
---
 crates/weavepy-vm/src/lib.rs                  | 191 +++++++++++-------
 .../test_attr_probes_and_lock_methods.py      | 187 +++++++++++++++++
 2 files changed, 306 insertions(+), 72 deletions(-)
 create mode 100644 tests/regrtest/test_attr_probes_and_lock_methods.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index badeeb5b..9fbaa219 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -15160,51 +15160,63 @@ impl Interpreter {
         let Some(callee_at) = n.checked_sub(argc + 2) else {
             return false;
         };
-        let (b, receiver) = match (&frame.stack[callee_at], &frame.stack[callee_at + 1]) {
-            (Object::Builtin(b), Object::Unbound) => (b, None),
-            (Object::Builtin(b), recv) => (b, Some(recv)),
+        if argc >= MAX_ARGS {
+            return false;
+        }
+        let recv_ok = |b: &crate::object::BuiltinFn, recv: &Object| {
+            b.binds_instance
+                && (!via_call
+                    || builtins::method_memo_tag(recv).is_some()
+                    || exception_init_call(b, recv)
+                    || (matches!(b.name, ".object_reduce_ex" | ".object_reduce")
+                        && matches!(recv, Object::Instance(_))))
+        };
+        let ok = match (&frame.stack[callee_at], &frame.stack[callee_at + 1]) {
+            (Object::Builtin(b), Object::Unbound) => !b.binds_instance,
+            (Object::Builtin(b), recv) => recv_ok(b, recv),
             (Object::BoundMethod(bm), Object::Unbound) if !bm.redispatch_descriptor => {
                 match &bm.function {
-                    Object::Builtin(b) => (b, Some(&bm.receiver)),
-                    _ => return false,
+                    Object::Builtin(b) => recv_ok(b, &bm.receiver),
+                    _ => false,
                 }
             }
-            _ => return false,
+            _ => false,
         };
-        let ok = argc < MAX_ARGS
-            && match receiver {
-                Some(recv) => {
-                    b.binds_instance
-                        && (!via_call
-                            || builtins::method_memo_tag(recv).is_some()
-                            || exception_init_call(b, recv)
-                            || (matches!(b.name, ".object_reduce_ex" | ".object_reduce")
-                                && matches!(recv, Object::Instance(_))))
-                }
-                None => !b.binds_instance,
-            };
         if !ok {
             return false;
         }
-        let (b, receiver) = (b.clone(), receiver.cloned());
-        // Committed. The receiver and the arguments move into `buf`; the
-        // self slot and the callable leave the stack.
+        // Committed. The callable and the self slot leave the stack, the
+        // receiver (the self slot's, moved, or a bound method's) and the
+        // arguments into `buf`.
         let mut buf = [const { std::mem::MaybeUninit::<Object>::uninit() }; MAX_ARGS + 1];
-        let first = usize::from(receiver.is_some());
-        if let Some(receiver) = receiver {
-            buf[0].write(receiver);
-        }
-        // SAFETY: the `argc` operands above the self slot move into `buf`
-        // (`argc < MAX_ARGS`) and the stack forgets them.
-        unsafe {
-            let src = frame.stack.as_ptr().add(callee_at + 2);
+        // SAFETY: the callable, the self slot and the `argc` operands
+        // above it leave the stack, which forgets them (`argc < MAX_ARGS`).
+        let (callable, moved_self) = unsafe {
+            let base = frame.stack.as_ptr().add(callee_at);
+            let callable = base.read();
+            let self_slot = base.add(1).read();
+            let moved_self = !matches!(self_slot, Object::Unbound);
+            if moved_self {
+                buf[0].write(self_slot);
+            } else if let Object::BoundMethod(bm) = &callable {
+                buf[0].write(bm.receiver.clone());
+            }
+            let first = usize::from(moved_self || matches!(callable, Object::BoundMethod(_)));
             for k in 0..argc {
-                buf[first + k].write(src.add(k).read());
+                buf[first + k].write(base.add(2 + k).read());
             }
-            frame.stack.set_len(callee_at + 2);
-        }
-        let self_slot = frame.stack.pop();
-        let callable = frame.stack.pop();
+            frame.stack.set_len(callee_at);
+            (callable, moved_self)
+        };
+        let b: &crate::object::BuiltinFn = match &callable {
+            Object::Builtin(b) => b,
+            Object::BoundMethod(bm) => match &bm.function {
+                Object::Builtin(b) => b,
+                _ => unreachable!("checked above"),
+            },
+            _ => unreachable!("checked above"),
+        };
+        let first = usize::from(moved_self || matches!(callable, Object::BoundMethod(_)));
         frame.pc = pc as u32 + 1;
         // SAFETY: the first `first + argc` entries were written above.
         let ops = unsafe {
@@ -15214,7 +15226,10 @@ impl Interpreter {
         let pending = self.core_pending_enter(sw, frame, pc);
         let result = if via_call {
             let globals = frame.globals.clone();
-            self.call(&Object::Builtin(b.clone()), ops, &[], &globals)
+            match &callable {
+                Object::Builtin(_) => self.call(&callable, ops, &[], &globals),
+                _ => self.call(&bm_builtin(&callable), ops, &[], &globals),
+            }
         } else {
             match b.call_kw.as_ref() {
                 Some(call_kw) => call_kw(ops, &[]),
@@ -15222,28 +15237,26 @@ impl Interpreter {
             }
         };
         self.core_pending_exit(sw, pending);
-        // The operands drop as the full handler's do, after the call.
+        // The operands drop as the full handler's do, after the call: the
+        // arguments plainly, the receiver the self slot held and the
+        // callable as the full handler releases them (a scalar or a shared
+        // builtin by a plain drop).
         // SAFETY: each initialized entry drops exactly once; `buf` itself
         // has no drop glue.
-        unsafe { std::ptr::drop_in_place(ops) };
-        for o in [callable, self_slot].into_iter().flatten() {
-            // A scalar or a shared builtin leaves by a plain drop; anything
-            // else as the full handler releases it.
-            if matches!(
-                o,
-                Object::Int(_)
-                    | Object::Float(_)
-                    | Object::Bool(_)
-                    | Object::None
-                    | Object::Unbound
-                    | Object::Builtin(_)
-            ) {
-                drop(o);
+        unsafe {
+            if moved_self {
+                let receiver = ops.as_ptr().read();
+                std::ptr::drop_in_place(&mut ops[1..]);
+                self.release(receiver);
             } else {
-                self.release(o);
+                std::ptr::drop_in_place(ops);
             }
         }
-        drop(b);
+        if matches!(callable, Object::Builtin(_)) {
+            drop(callable);
+        } else {
+            self.release(callable);
+        }
         match result {
             Ok(v) => {
                 // SAFETY: as above (the body ran nested activations of its
@@ -26251,18 +26264,35 @@ impl Interpreter {
     /// the default. `None` for anything else (descriptors, dunders, a
     /// missing attribute's `AttributeError`), which the full call handles.
     fn leaf_getattr(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
-        let (recv, Object::Str(name_obj)) = (args.first()?, args.get(1)?) else {
+        if args.len() > 3 {
+            return None;
+        }
+        Self::leaf_getattr_parts(args.first()?, args.get(1)?, args.get(2), false).map(Ok)
+    }
+
+    /// [`Self::leaf_getattr`] on its parts, `default` the third argument's
+    /// value if any. With `probe` (`hasattr`), the value found matters
+    /// only as present: a method found to bind is answered with `True`
+    /// instead of a bound method, and a built-in kind's method (see
+    /// [`builtins::lookup_method`]) is found without binding it.
+    fn leaf_getattr_parts(
+        recv: &Object,
+        name_arg: &Object,
+        default: Option<&Object>,
+        probe: bool,
+    ) -> Option<Object> {
+        let Object::Str(name_obj) = name_arg else {
             return None;
         };
         let name: &str = name_obj;
-        if args.len() > 3 || crate::object::exotic_str_keys_possible() {
+        if crate::object::exotic_str_keys_possible() {
             return None;
         }
         let inst = match recv {
             Object::Instance(inst) => inst,
             // A function's stored attribute (`__doc__` once read,
             // `__wrapped__`).
-            Object::Function(f) => return function_stored_attr(f, name_obj).map(Ok),
+            Object::Function(f) => return function_stored_attr(f, name_obj),
             // A plain module's global (`hasattr(os, 'getpid')`): its
             // dictionary's value (`ModuleType`'s descriptors are dunders;
             // a missing name may reach a module `__getattr__`).
@@ -26272,19 +26302,28 @@ impl Interpreter {
                 return m
                     .dict
                     .borrow()
-                    .get(&DictKey(args[1].clone()))
-                    .cloned()
-                    .map(Ok);
+                    .get(&crate::object::StrKey(name))
+                    .cloned();
+            }
+            // A built-in kind's method (`hasattr(stream, "write")`): its
+            // lookup runs no code, and a found method means the attribute
+            // is there.
+            _ if probe
+                && !name.starts_with('_')
+                && builtins::method_memo_tag(recv).is_some()
+                && builtins::lookup_method(recv, name).is_some() =>
+            {
+                return Some(Object::Bool(true));
             }
             _ => return None,
         };
         // A dunder nothing supplies (`copy`'s `__deepcopy__` probe): the
         // default.
         if name.starts_with("__") {
-            if args.len() == 3 && attr_certainly_missing(&args[0], name) {
-                return Some(Ok(args[2].clone()));
+            if let Some(d) = default.filter(|_| attr_certainly_missing(recv, name)) {
+                return Some(d.clone());
             }
-            return Self::leaf_builtin_dunder_attr(inst, &args[0], name).map(Ok);
+            return Self::leaf_builtin_dunder_attr(inst, recv, name);
         }
         let cls = inst.cls_raw();
         if cls.native_kind.get() != 0
@@ -26312,18 +26351,19 @@ impl Interpreter {
             return None;
         }
         if let Some(v) = inst.attr_get_str(name) {
-            return Some(Ok(v));
+            return Some(v);
         }
-        Some(Ok(match on_class {
+        Some(match on_class {
+            Some(Object::Function(_)) if probe => Object::Bool(true),
             Some(f @ Object::Function(_)) => {
-                Object::BoundMethod(Rc::new(BoundMethod::new(args[0].clone(), f)))
+                Object::BoundMethod(Rc::new(BoundMethod::new(recv.clone(), f)))
             }
             Some(v) => v,
-            None if args.len() == 3 && !cls.dunder(crate::types::Dunder::GetAttr).present() => {
-                args[2].clone()
+            None if default.is_some() && !cls.dunder(crate::types::Dunder::GetAttr).present() => {
+                default?.clone()
             }
             None => return None,
-        }))
+        })
     }
 
     /// A dunder a plain instance gets from a builtin method on its class
@@ -26371,11 +26411,9 @@ impl Interpreter {
         let [obj, name] = args else {
             return None;
         };
-        let probe = [obj.clone(), name.clone(), Object::Unbound];
-        match Self::leaf_getattr(&probe)? {
-            Ok(Object::Unbound) => Some(Ok(Object::Bool(false))),
-            Ok(_) => Some(Ok(Object::Bool(true))),
-            Err(_) => None,
+        match Self::leaf_getattr_parts(obj, name, Some(&Object::Unbound), true)? {
+            Object::Unbound => Some(Ok(Object::Bool(false))),
+            _ => Some(Ok(Object::Bool(true))),
         }
     }
 
@@ -71614,6 +71652,15 @@ fn exception_init_call(b: &crate::object::BuiltinFn, recv: &Object) -> bool {
     b.name == "__init__" && matches!(recv, Object::Instance(i) if i.cls_raw().flags.is_exception)
 }
 
+/// The builtin function a bound method of one wraps, as its own callable.
+#[cold]
+fn bm_builtin(callable: &Object) -> Object {
+    match callable {
+        Object::BoundMethod(bm) => bm.function.clone(),
+        other => other.clone(),
+    }
+}
+
 /// Builtin type methods the interpreter dispatches by name (their bodies
 /// iterate or compare through it): the core loop calls one on its exact
 /// builtin receiver through [`Interpreter::call`] (see
diff --git a/tests/regrtest/test_attr_probes_and_lock_methods.py b/tests/regrtest/test_attr_probes_and_lock_methods.py
new file mode 100644
index 00000000..fc254c85
--- /dev/null
+++ b/tests/regrtest/test_attr_probes_and_lock_methods.py
@@ -0,0 +1,187 @@
+"""`hasattr`/`getattr` probes, builtin method calls and lock methods.
+
+`hasattr` answers a method on a built-in kind or a class without
+binding it, and must still propagate anything but `AttributeError`.
+Builtin method calls release their receiver promptly, and a lock's
+context-manager methods find their per-instance implementation by a
+remembered position, which must survive other entries changing.
+"""
+
+import io
+import threading
+import unittest
+
+
+class Finalized:
+    count = 0
+
+    def __del__(self):
+        Finalized.count += 1
+
+
+class Props:
+    def method(self):
+        return 1
+
+    @property
+    def missing(self):
+        raise AttributeError("missing")
+
+    @property
+    def broken(self):
+        raise ValueError("broken")
+
+
+class Dynamic:
+    def __getattr__(self, name):
+        if name == "boom":
+            raise KeyError(name)
+        if name.startswith("dyn_"):
+            return name
+        raise AttributeError(name)
+
+
+class Failing(Exception):
+    def __init__(self, *args):
+        super().__init__(*args)
+
+
+class AttrProbeTest(unittest.TestCase):
+    def test_stream_methods(self):
+        for s in (io.StringIO(), io.BytesIO()):
+            for _ in range(500):
+                self.assertTrue(hasattr(s, "write"))
+                self.assertTrue(hasattr(s, "getvalue"))
+                self.assertFalse(hasattr(s, "no_such_method"))
+            s.close()
+            self.assertTrue(hasattr(s, "write"))
+            self.assertTrue(getattr(s, "closed"))
+
+    def test_builtin_kinds(self):
+        for _ in range(500):
+            self.assertTrue(hasattr("x", "upper"))
+            self.assertTrue(hasattr([], "append"))
+            self.assertTrue(hasattr({}, "get"))
+            self.assertFalse(hasattr("x", "append"))
+            self.assertTrue(hasattr(1, "real"))
+            self.assertTrue(hasattr(1.5, "is_integer"))
+            self.assertFalse(hasattr((), "_private"))
+        self.assertEqual(getattr("abc", "upper")(), "ABC")
+        self.assertIsNone(getattr("abc", "nope", None))
+
+    def test_instances(self):
+        p = Props()
+        for _ in range(500):
+            self.assertTrue(hasattr(p, "method"))
+            self.assertFalse(hasattr(p, "missing"))
+            self.assertFalse(hasattr(p, "nothing"))
+            self.assertEqual(getattr(p, "method")(), 1)
+            self.assertEqual(getattr(p, "nothing", 7), 7)
+        with self.assertRaises(ValueError):
+            hasattr(p, "broken")
+        p.method = 5
+        self.assertTrue(hasattr(p, "method"))
+        self.assertEqual(getattr(p, "method"), 5)
+        del p.method
+        self.assertEqual(getattr(p, "method")(), 1)
+
+    def test_getattr_hook(self):
+        d = Dynamic()
+        for _ in range(300):
+            self.assertTrue(hasattr(d, "dyn_x"))
+            self.assertFalse(hasattr(d, "other"))
+        with self.assertRaises(KeyError):
+            hasattr(d, "boom")
+        with self.assertRaises(TypeError):
+            hasattr(d, 1)
+
+    def test_modules_and_functions(self):
+        import os
+
+        def f():
+            pass
+
+        f.tag = "t"
+        for _ in range(300):
+            self.assertTrue(hasattr(os, "getpid"))
+            self.assertFalse(hasattr(os, "no_such_name_here"))
+            self.assertTrue(hasattr(f, "tag"))
+            self.assertEqual(getattr(f, "tag", None), "t")
+            self.assertIsNone(getattr(f, "untagged", None))
+
+
+class BuiltinLaneTest(unittest.TestCase):
+    def test_receiver_released_promptly(self):
+        Finalized.count = 0
+        for i in range(300):
+            Finalized().__reduce_ex__(4)
+            self.assertEqual(Finalized.count, i + 1)
+
+    def test_exception_init_through_super(self):
+        for i in range(300):
+            e = Failing(i, "x")
+            self.assertEqual(e.args, (i, "x"))
+
+    def test_bound_builtin_methods(self):
+        items = []
+        append = items.append
+        for i in range(300):
+            append(i)
+            self.assertEqual("-".join(["a", str(i)]), "a-%d" % i)
+        self.assertEqual(len(items), 300)
+
+
+class LockMethodTest(unittest.TestCase):
+    def check_lock(self, lock):
+        for _ in range(500):
+            with lock:
+                self.assertTrue(lock.locked() if hasattr(lock, "locked") else True)
+        self.assertFalse(lock.locked())
+        try:
+            with lock:
+                raise RuntimeError("inside")
+        except RuntimeError:
+            pass
+        self.assertFalse(lock.locked())
+        self.assertTrue(lock.acquire(False))
+        lock.release()
+
+    def test_lock(self):
+        self.check_lock(threading.Lock())
+
+    def test_rlock(self):
+        r = threading.RLock()
+        self.check_lock(r)
+        with r:
+            with r:
+                self.assertTrue(r._is_owned())
+        self.assertFalse(r._is_owned())
+
+    def test_many_locks(self):
+        locks = [threading.Lock() for _ in range(50)] + [threading.RLock() for _ in range(50)]
+        for _ in range(20):
+            for lock in locks:
+                with lock:
+                    pass
+                self.assertFalse(lock.locked())
+
+    def test_condition_and_threads(self):
+        cond = threading.Condition()
+        box = []
+
+        def worker():
+            with cond:
+                box.append(1)
+                cond.notify()
+
+        with cond:
+            t = threading.Thread(target=worker)
+            t.start()
+            while not box:
+                cond.wait(5)
+        t.join()
+        self.assertEqual(box, [1])
+
+
+if __name__ == "__main__":
+    unittest.main()

From 2f9321d2fc9bd9cf8fdc3d7cc9bd97b26451bc11 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 10:39:29 -0700
Subject: [PATCH 081/173] perf: read a lock's method entry without a dict
 borrow

The lock trampolines' remembered-position read now views the lock's
published dict in place under the GIL instead of creating it on demand
and taking a borrow guard, and compares the key directly; a miss still
takes the borrowed, hashed probe.

context_managers -0.6% instructions (callgrind).
---
 crates/weavepy-vm/src/stdlib/thread_real.rs | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs
index 40c28656..bb3ac79a 100644
--- a/crates/weavepy-vm/src/stdlib/thread_real.rs
+++ b/crates/weavepy-vm/src/stdlib/thread_real.rs
@@ -381,14 +381,17 @@ fn instance_closure(
     site: &ClosureSite,
 ) -> Option<Object> {
     use std::sync::atomic::Ordering::Relaxed;
-    let d = inst.dict_cell().borrow();
-    if let Some((DictKey(Object::Str(k)), v)) = d.get_index(site.at.load(Relaxed)) {
-        let k: &str = k;
-        // (The dict's keys are the same static names.)
-        if std::ptr::eq(k.as_ptr(), name.as_ptr()) && k.len() == name.len() || k == name {
-            return Some(v.clone());
+    let at = site.at.load(Relaxed);
+    // SAFETY: a read under the GIL that ends before anything else runs.
+    if let Some(d) = inst.dict.published().and_then(|d| unsafe { d.peek() }) {
+        if let Some((DictKey(Object::Str(k)), v)) = d.get_index(at) {
+            let k: &str = k;
+            if k == name {
+                return Some(v.clone());
+            }
         }
     }
+    let d = inst.dict_cell().borrow();
     let mut h = site.hash.load(Relaxed);
     if h == 0 {
         h = crate::object::py_str_hash(name);

From a019437a2c73a4c01d9ad3b8251bde67ef6b9196 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 11:09:43 -0700
Subject: [PATCH 082/173] perf: pool instances a compiled loop discards

The core loop's release pools a dying plain instance for the next
allocation (CPython's per-type freelist), but compiled code's `POP_TOP`
dropped it plainly, so `C(...)` as a statement freed the instance (and
a slotted one's member array) and allocated a fresh one next time. The
frame JIT's pop now pools such an instance the same way.

`PyInstance::recycle` reports an instance it doesn't pool instead of
dropping it, and the pool push tolerates a thread whose locals are
being torn down.

slotted `C(i, 2)` statement: 1811 -> 1611 instructions.
---
 crates/weavepy-vm/src/frame_jit.rs            | 11 +++
 crates/weavepy-vm/src/types.rs                | 43 +++++++---
 .../regrtest/test_discarded_instance_pool.py  | 85 +++++++++++++++++++
 3 files changed, 125 insertions(+), 14 deletions(-)
 create mode 100644 tests/regrtest/test_discarded_instance_pool.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index f9bd8b00..37d04e60 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1146,6 +1146,17 @@ unsafe extern "C" fn h_pop(slot: *mut Object) -> u32 {
         if !droppable(&*slot) {
             return 1;
         }
+        // A plain instance dying here goes to the instance pool, as the
+        // core loop's release sends it (`C(...)` as a statement).
+        if let Object::Instance(i) = &*slot {
+            if Rc::strong_count(i) == 1 && i.dies_by_plain_drop() {
+                let Object::Instance(i) = slot.read() else {
+                    unreachable!("matched above")
+                };
+                crate::types::PyInstance::try_recycle(i);
+                return 0;
+            }
+        }
         crate::drop_hot(slot.read());
     }
     0
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 1e023802..442c7de9 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -3645,20 +3645,27 @@ impl PyInstance {
     /// pool for `new_deferred` (CPython's per-type freelist), or drop it.
     /// The caller established that `obj` is its only reference, the
     /// instance is still deferred and owns no C body.
-    pub fn try_recycle(mut inst: Rc<Self>) {
+    pub fn try_recycle(inst: Rc<Self>) {
+        if let Err(inst) = Self::recycle(inst) {
+            // (A plain drop: the caller found no finalizer owed.)
+            crate::rc::drop_arc(Rc::into_arc(inst));
+        }
+    }
+
+    /// [`Self::try_recycle`], handing back an instance it doesn't pool
+    /// (whose state may be partly reset: it is dying) for the caller to
+    /// free without its last-release hook, which may be what called this.
+    pub(crate) fn recycle(mut inst: Rc<Self>) -> Result<(), Rc<Self>> {
         // SAFETY: the caller holds the only reference.
         if unsafe { &*inst.class.as_ptr() }.native_kind.get() != 0 {
-            if let Err(inst) = crate::stdlib::datetime_native::recycle(inst) {
-                drop(Rc::into_arc(inst));
-            }
-            return;
+            return crate::stdlib::datetime_native::recycle(inst);
         }
         let Some(m) = Rc::get_mut(&mut inst) else {
-            return;
+            return Err(inst);
         };
         let owner = std::ptr::from_ref(&*m) as usize;
         if m.native.get().is_some() || m.finalize_ran.get() || m.c_body.get() != 0 {
-            return;
+            return Err(inst);
         }
         // Split values are atomic (the instance is deferred): clearing
         // them runs no code. The allocation stays for the next tenant.
@@ -3671,15 +3678,16 @@ impl PyInstance {
             // hand out the same `Arc`, and a holder must keep seeing the
             // dead instance's attributes, not the next tenant's.
             if m.dict.strong_count() != 1 {
-                return;
+                return Err(inst);
             }
             // Reset in place: values are atomic, so clearing runs no code
             // that could observe the instance.
             let Ok(mut d) = dict.try_borrow_mut() else {
-                return;
+                return Err(inst);
             };
             if d.capacity() > 32 {
-                return;
+                drop(d);
+                return Err(inst);
             }
             d.map_mut_atomic_store().clear();
             d.reset_deferred_owner(owner);
@@ -3691,12 +3699,19 @@ impl PyInstance {
         // A pooled instance must not keep its class alive (a transient
         // class is collected as soon as its last instance dies).
         *m.class.get_mut() = crate::builtin_types::builtin_types().object_.clone();
-        INSTANCE_POOL.with(|p| {
-            let mut p = p.borrow_mut();
-            if p.len() < INSTANCE_POOL_CAP {
-                p.push(inst);
+        let mut inst = Some(inst);
+        // (A thread whose locals are being torn down pools nothing.)
+        let _ = INSTANCE_POOL.try_with(|p| {
+            if let Ok(mut p) = p.try_borrow_mut() {
+                if p.len() < INSTANCE_POOL_CAP {
+                    p.extend(inst.take());
+                }
             }
         });
+        match inst {
+            None => Ok(()),
+            Some(inst) => Err(inst),
+        }
     }
 
     /// Whether tracking is still deferred (the `__dict__` carries the
diff --git a/tests/regrtest/test_discarded_instance_pool.py b/tests/regrtest/test_discarded_instance_pool.py
new file mode 100644
index 00000000..d77b68e7
--- /dev/null
+++ b/tests/regrtest/test_discarded_instance_pool.py
@@ -0,0 +1,85 @@
+"""Instances discarded by a statement (`C(...)` whose value is unused).
+
+A plain instance that dies as a compiled loop pops it goes back to the
+instance pool. One owing a finalizer, one with a weak reference or one
+another owner still holds must not be pooled, and a pooled instance
+reused for another class must carry nothing of its last tenant.
+"""
+
+import gc
+import unittest
+import weakref
+
+
+class Plain:
+    def __init__(self, a):
+        self.a = a
+        self.b = "b"
+
+
+class Slotted:
+    __slots__ = ("x", "y")
+
+    def __init__(self, x):
+        self.x = x
+
+
+class Final:
+    log = []
+
+    def __init__(self, tag):
+        self.tag = tag
+
+    def __del__(self):
+        Final.log.append(self.tag)
+
+
+class Weak:
+    __slots__ = ("v", "__weakref__")
+
+    def __init__(self, v):
+        self.v = v
+
+
+class PoolTest(unittest.TestCase):
+    def test_discarded_in_a_loop(self):
+        for i in range(3000):
+            Plain(i)
+            Slotted(i)
+        p = Plain(1)
+        self.assertEqual(vars(p), {"a": 1, "b": "b"})
+        s = Slotted.__new__(Slotted)
+        self.assertFalse(hasattr(s, "x"))
+        self.assertFalse(hasattr(s, "y"))
+        e = Plain.__new__(Plain)
+        self.assertEqual(vars(e), {})
+
+    def test_finalizer_runs_at_the_statement(self):
+        Final.log.clear()
+        for i in range(500):
+            Final(i)
+            self.assertEqual(Final.log[-1], i)
+        self.assertEqual(len(Final.log), 500)
+
+    def test_weakref_callback(self):
+        fired = []
+        refs = []
+        for i in range(300):
+            w = Weak(i)
+            refs.append(weakref.ref(w, lambda r, i=i: fired.append(i)))
+            del w
+        self.assertEqual(fired, list(range(300)))
+        self.assertTrue(all(r() is None for r in refs))
+
+    def test_shared_owner_survives(self):
+        keep = []
+        for i in range(1000):
+            keep.append(Plain(i))
+            Plain(-i)
+        self.assertEqual([k.a for k in keep], list(range(1000)))
+        gc.collect()
+        self.assertEqual(keep[500].b, "b")
+
+
+if __name__ == "__main__":
+    unittest.main()

From 0b7bd04dbb2c8004dbc82f8b550d040f4008a6e8 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 11:50:02 -0700
Subject: [PATCH 083/173] perf: build a materialized instance dict in one pass

An instance's split attributes become its `__dict__` the first time
something needs the real dictionary (`vars(obj)`, logging's
`record.__dict__` for every formatted record). Each attribute went in
through a probing insert into a table sized up front. The names are
distinct and their hashes kept, so the entries are now laid down in
order and the index built once.

Also: the builtin call lane parks its activation through the fast
pending path, and a lock trampoline compares its entry's short name in
line.

vars() of a 20-attribute instance and a %-format over it: 16264 -> 15200.
---
 crates/weavepy-vm/src/dictmap.rs              | 22 +++++++
 crates/weavepy-vm/src/inst_dict.rs            | 45 ++++++++-----
 crates/weavepy-vm/src/lib.rs                  |  2 +-
 crates/weavepy-vm/src/object.rs               | 11 ++++
 crates/weavepy-vm/src/stdlib/thread_real.rs   |  5 +-
 .../test_instance_dict_materialize.py         | 66 +++++++++++++++++++
 6 files changed, 130 insertions(+), 21 deletions(-)
 create mode 100644 tests/regrtest/test_instance_dict_materialize.py

diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index e55f611b..f7e6d72c 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -215,6 +215,28 @@ impl DictMap {
         m
     }
 
+    /// A table of the entries `items` yields, whose keys are distinct and
+    /// each paired with its Python hash, built in one pass: the entries
+    /// are laid down in order, then indexed once (no probe compares a
+    /// key).
+    pub fn from_unique_hashed(
+        items: impl ExactSizeIterator<Item = (i64, DictKey, Object)>,
+    ) -> Self {
+        let mut entries = Vec::with_capacity(items.len());
+        for (hash, key, value) in items {
+            entries.push(Bucket { hash, key, value });
+        }
+        let mut m = Self {
+            entries,
+            slots: Box::new([]),
+            shift: 64,
+        };
+        if m.entries.len() > SMALL {
+            m.rebuild_slots(slots_for(m.entries.len()));
+        }
+        m
+    }
+
     /// [`Self::with_capacity`]; the hasher is implied (the table mixes
     /// Python hashes itself).
     #[inline]
diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index 41ab8786..f2121130 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -1151,26 +1151,35 @@ impl InstDict {
                 s.snapshot()
             }
         };
-        let n = values.len();
-        let mut d = if owner.deferred.get() {
-            DictData::deferred_for_capacity(std::ptr::from_ref(owner) as usize, n)
+        // A deferred owner holds only atomic values, and a tracked one
+        // needs no barrier: the table is built without either. The names
+        // are distinct `str`s whose hashes the shared table kept, so the
+        // entries go down in order and are indexed once, without hashing
+        // or comparing.
+        let map = match keys {
+            Some(keys) => {
+                let n = values.len().min(keys.len());
+                crate::dictmap::DictMap::from_unique_hashed(
+                    values.into_iter().take(n).enumerate().map(|(i, v)| {
+                        // SAFETY: slot `i` is published (below the length).
+                        let (k, hash) = unsafe {
+                            (
+                                (*keys.keys[i].get()).assume_init_ref(),
+                                *keys.hashes[i].get(),
+                            )
+                        };
+                        (hash, k.clone(), v)
+                    }),
+                )
+            }
+            None => crate::dictmap::DictMap::default(),
+        };
+        let deferred = if owner.deferred.get() {
+            std::ptr::from_ref(owner) as usize
         } else {
-            DictData::with_capacity_and_hasher(n, crate::fasthash::FxBuildHasher)
+            0
         };
-        if let Some(keys) = keys {
-            // A deferred owner holds only atomic values, and a tracked
-            // one needs no barrier: insert without either. The names are
-            // distinct `str`s whose hashes the table kept, so each goes
-            // straight into its bucket, without hashing or comparing.
-            let map = d.map_mut_atomic_store();
-            for (i, v) in values.into_iter().enumerate() {
-                if let Some(k) = keys.get(i) {
-                    // SAFETY: slot `i` is published (`get` found its name).
-                    let hash = unsafe { *keys.hashes[i].get() };
-                    map.insert_unique_hashed(hash, k.clone(), v);
-                }
-            }
-        }
+        let d = DictData::from_map_for(deferred, map);
         self.lazy.get_or_init(|| Rc::new(RefCell::new(d)))
     }
 }
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 9fbaa219..ded57740 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -15223,7 +15223,7 @@ impl Interpreter {
             std::slice::from_raw_parts_mut(buf.as_mut_ptr().cast::<Object>(), first + argc)
         };
         // The body may run Python code that looks up the stack.
-        let pending = self.core_pending_enter(sw, frame, pc);
+        let pending = self.core_pending_enter_fast(sw, frame, pc);
         let result = if via_call {
             let globals = frame.globals.clone();
             match &callable {
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 5649d9bb..e33b02ec 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -4784,6 +4784,17 @@ impl DictData {
         }
     }
 
+    /// An instance `__dict__` holding `map`, guarding the deferred
+    /// tracking of the instance at `owner` (`0`: none).
+    pub(crate) fn from_map_for(owner: usize, map: DictMap) -> Self {
+        Self {
+            map,
+            stamp: next_dict_stamp(),
+            deferred_owner: std::sync::atomic::AtomicUsize::new(owner),
+            key_filter: std::sync::atomic::AtomicU64::new(0),
+        }
+    }
+
     /// Re-arm the owner record on a recycled (empty) instance dict.
     #[inline]
     pub(crate) fn reset_deferred_owner(&mut self, owner: usize) {
diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs
index bb3ac79a..1aba095a 100644
--- a/crates/weavepy-vm/src/stdlib/thread_real.rs
+++ b/crates/weavepy-vm/src/stdlib/thread_real.rs
@@ -385,8 +385,9 @@ fn instance_closure(
     // SAFETY: a read under the GIL that ends before anything else runs.
     if let Some(d) = inst.dict.published().and_then(|d| unsafe { d.peek() }) {
         if let Some((DictKey(Object::Str(k)), v)) = d.get_index(at) {
-            let k: &str = k;
-            if k == name {
+            // (A short name compared in line, without `memcmp`.)
+            let (k, n) = (k.as_bytes(), name.as_bytes());
+            if k.len() == n.len() && k.iter().zip(n).all(|(a, b)| a == b) {
                 return Some(v.clone());
             }
         }
diff --git a/tests/regrtest/test_instance_dict_materialize.py b/tests/regrtest/test_instance_dict_materialize.py
new file mode 100644
index 00000000..725d9f8c
--- /dev/null
+++ b/tests/regrtest/test_instance_dict_materialize.py
@@ -0,0 +1,66 @@
+"""An instance's split attributes becoming its real `__dict__`.
+
+The dictionary is built in one pass from the split values: its order,
+lookups (small and large tables), later stores, deletes and identity
+must be exactly a dictionary's.
+"""
+
+import unittest
+
+
+def make(n, tag=0):
+    class C:
+        pass
+
+    objs = []
+    for k in range(3):
+        o = C()
+        for i in range(n):
+            setattr(o, "a%d" % i, (tag, k, i))
+        objs.append(o)
+    return objs
+
+
+class MaterializeTest(unittest.TestCase):
+    def check(self, n):
+        for o in make(n, n):
+            d = o.__dict__
+            self.assertIs(o.__dict__, d)
+            names = ["a%d" % i for i in range(n)]
+            self.assertEqual(list(d), names)
+            for i, name in enumerate(names):
+                self.assertEqual(d[name][2], i)
+                self.assertIn(name, d)
+            self.assertNotIn("missing", d)
+            o.extra = 1
+            self.assertEqual(d["extra"], 1)
+            if n:
+                del o.a0
+                self.assertNotIn("a0", d)
+                d["a0"] = "back"
+                self.assertEqual(o.a0, "back")
+                self.assertEqual(list(d)[-1], "a0")
+            self.assertEqual(len(d), n + 1)
+            self.assertEqual("%(extra)s" % d, "1")
+
+    def test_sizes(self):
+        for n in (0, 1, 7, 8, 9, 15, 16, 17, 29, 30, 31, 40):
+            self.check(n)
+
+    def test_vars_and_hash_collisions(self):
+        class K:
+            pass
+
+        k = K()
+        # Names whose hashes share low bits still land and resolve.
+        names = ["x%d" % i for i in range(64)]
+        for n in names:
+            setattr(k, n, n)
+        v = vars(k)
+        self.assertEqual(sorted(v), sorted(names))
+        for n in names:
+            self.assertEqual(v[n], n)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 0132b92fdb88201c26688e9d1e53b8e1a66a0dc3 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 12:15:58 -0700
Subject: [PATCH 084/173] perf: run wider store-only __init__ bodies by their
 stores

A constructor that only stores its parameters and constants into
`self` is run by its stores alone, but only up to sixteen of them; a
wider one (twenty attributes) fell back to the frameless evaluator's
buffered stores, about 180 instructions each. The limit is now the
number of names a class shares for its instances (thirty), which the
split layout already bounds.

The materialized `__dict__` build also fills its entries by index
instead of through an iterator chain.

20-attribute constructor: 5827 -> 4102 instructions; vars() of one
and a %-format over it: 15200 -> 12397.
---
 crates/weavepy-vm/src/dictmap.rs       | 22 ++++++---
 crates/weavepy-vm/src/inst_dict.rs     | 32 +++++++-----
 crates/weavepy-vm/src/lib.rs           |  7 ++-
 tests/regrtest/test_wide_store_init.py | 67 ++++++++++++++++++++++++++
 4 files changed, 107 insertions(+), 21 deletions(-)
 create mode 100644 tests/regrtest/test_wide_store_init.py

diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index f7e6d72c..c93f184a 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -215,17 +215,25 @@ impl DictMap {
         m
     }
 
-    /// A table of the entries `items` yields, whose keys are distinct and
-    /// each paired with its Python hash, built in one pass: the entries
-    /// are laid down in order, then indexed once (no probe compares a
+    /// A table of the `n` entries `entry(i)` gives in order, whose keys are
+    /// distinct and each paired with its Python hash, built in one pass:
+    /// the entries are laid down, then indexed once (no probe compares a
     /// key).
+    #[inline]
     pub fn from_unique_hashed(
-        items: impl ExactSizeIterator<Item = (i64, DictKey, Object)>,
+        n: usize,
+        mut entry: impl FnMut(usize) -> (i64, DictKey, Object),
     ) -> Self {
-        let mut entries = Vec::with_capacity(items.len());
-        for (hash, key, value) in items {
-            entries.push(Bucket { hash, key, value });
+        let mut entries: Vec<Bucket> = Vec::with_capacity(n);
+        let dst = entries.as_mut_ptr();
+        for i in 0..n {
+            let (hash, key, value) = entry(i);
+            // SAFETY: `i < n`, inside the capacity; the length is set once
+            // every slot below `n` is written.
+            unsafe { dst.add(i).write(Bucket { hash, key, value }) };
         }
+        // SAFETY: the first `n` buckets were written above.
+        unsafe { entries.set_len(n) };
         let mut m = Self {
             entries,
             slots: Box::new([]),
diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index f2121130..0eb1fa44 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -1156,24 +1156,32 @@ impl InstDict {
         // are distinct `str`s whose hashes the shared table kept, so the
         // entries go down in order and are indexed once, without hashing
         // or comparing.
+        let mut values = values;
         let map = match keys {
             Some(keys) => {
                 let n = values.len().min(keys.len());
-                crate::dictmap::DictMap::from_unique_hashed(
-                    values.into_iter().take(n).enumerate().map(|(i, v)| {
-                        // SAFETY: slot `i` is published (below the length).
-                        let (k, hash) = unsafe {
-                            (
-                                (*keys.keys[i].get()).assume_init_ref(),
-                                *keys.hashes[i].get(),
-                            )
-                        };
-                        (hash, k.clone(), v)
-                    }),
-                )
+                let src = values.as_ptr();
+                let map = crate::dictmap::DictMap::from_unique_hashed(n, |i| {
+                    // SAFETY: slot `i` is published (below the length),
+                    // and value `i` moves out once (the vector forgets the
+                    // first `n` below).
+                    unsafe {
+                        let k = (*keys.keys[i].get()).assume_init_ref();
+                        (*keys.hashes[i].get(), k.clone(), src.add(i).read())
+                    }
+                });
+                // SAFETY: the first `n` values moved into the table; any
+                // past the names (none in practice) drop with the vector.
+                unsafe {
+                    let rest = values.len() - n;
+                    std::ptr::copy(src.add(n), values.as_mut_ptr(), rest);
+                    values.set_len(rest);
+                }
+                map
             }
             None => crate::dictmap::DictMap::default(),
         };
+        drop(values);
         let deferred = if owner.deferred.get() {
             std::ptr::from_ref(owner) as usize
         } else {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ded57740..5354c484 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -17314,7 +17314,7 @@ impl Interpreter {
         // layout positions the stores then fill. A site that recorded
         // anything else for this version never will.
         let laid_out = ty.slot_layout.get().and_then(Option::as_ref);
-        let mut members = [0u8; 16];
+        let mut members = [0u8; crate::inst_dict::SHARED_KEYS_CAP];
         if laid_out.is_some() {
             let mut seen = 0u64;
             for (k, &(pc, _)) in shape.stores.iter().enumerate() {
@@ -74199,7 +74199,10 @@ fn store_init_shape(code: &CodeObject) -> Option<StoreInit> {
             {
                 // The final `return None`: the shape holds when nothing
                 // else follows it.
-                if pc + 2 != ins.len() || stores.is_empty() || stores.len() > 16 {
+                if pc + 2 != ins.len()
+                    || stores.is_empty()
+                    || stores.len() > crate::inst_dict::SHARED_KEYS_CAP
+                {
                     return None;
                 }
                 return Some(StoreInit {
diff --git a/tests/regrtest/test_wide_store_init.py b/tests/regrtest/test_wide_store_init.py
new file mode 100644
index 00000000..d6899dbc
--- /dev/null
+++ b/tests/regrtest/test_wide_store_init.py
@@ -0,0 +1,67 @@
+"""Store-only `__init__` bodies with many attributes.
+
+A constructor that only stores its parameters and constants into `self`
+is run by its stores alone, up to as many attributes as a class shares
+names for. Wide ones (more than sixteen stores), slotted ones and ones
+past the limit must all build exactly the instance the body would.
+"""
+
+import unittest
+
+NAMES = ["f%d" % i for i in range(34)]
+
+
+def make_class(n, slots=False):
+    params = ", ".join("a%d" % i for i in range(min(n, 7)))
+    body = "\n".join(
+        "        self.%s = %s" % (NAMES[i], "a%d" % (i % 7) if i % 3 else repr(i))
+        for i in range(n)
+    )
+    src = "class C:\n"
+    if slots:
+        src += "    __slots__ = %r\n" % (tuple(NAMES[:n]),)
+    src += "    def __init__(self, %s):\n%s\n" % (params, body)
+    ns = {}
+    exec(src, ns)
+    return ns["C"], min(n, 7)
+
+
+def expected(n):
+    return {NAMES[i]: (i % 7 if i % 3 else i) for i in range(n)}
+
+
+class WideInitTest(unittest.TestCase):
+    def check(self, n, slots):
+        cls, nparams = make_class(n, slots)
+        args = list(range(nparams))
+        for _ in range(300):
+            obj = cls(*args)
+        if slots:
+            got = {name: getattr(obj, name) for name in NAMES[:n]}
+            self.assertFalse(hasattr(obj, "__dict__"))
+        else:
+            got = vars(obj)
+            self.assertEqual(list(got), NAMES[:n])
+        self.assertEqual(got, expected(n))
+
+    def test_widths(self):
+        for n in (1, 15, 16, 17, 20, 29, 30, 31, 33):
+            for slots in (False, True):
+                with self.subTest(n=n, slots=slots):
+                    self.check(n, slots)
+
+    def test_class_change_between_calls(self):
+        cls, nparams = make_class(20)
+        for _ in range(300):
+            obj = cls(*range(nparams))
+        cls.f3 = property(lambda self: "prop", lambda self, v: None)
+        obj = cls(*range(nparams))
+        self.assertEqual(obj.f3, "prop")
+        self.assertNotIn("f3", vars(obj))
+        del cls.f3
+        obj = cls(*range(nparams))
+        self.assertEqual(obj.f3, 3)
+
+
+if __name__ == "__main__":
+    unittest.main()

From ac9d066f16eecf3c13caa8025ca478cf31026ca3 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 12:42:45 -0700
Subject: [PATCH 085/173] perf: store a split attribute from one view of the
 values

An attribute store through a site's field shortcut first tried the
overwrite (finding the class's names, running the write barrier and
viewing the split values) and, for the constructor shape, then did all
of it again to append. `PyInstance::split_store_at` decides between
the two from one view, keeping the overwrite's droppable check (a
refused old value still declines the whole store) and the append's
checks.

A ten-attribute `__init__` with calls in it: 5435 -> 5169 instructions.
---
 crates/weavepy-vm/src/inst_dict.rs | 44 ++++++++++++++++++++++++++++++
 crates/weavepy-vm/src/lib.rs       | 31 ++++++++++-----------
 2 files changed, 58 insertions(+), 17 deletions(-)

diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index 0eb1fa44..dbda5e07 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -1274,6 +1274,50 @@ impl crate::types::PyInstance {
         split.append_over(keys, || cls.shared_keys.share(), i, value)
     }
 
+    /// Store `value` at position `i` of the class's shared names from one
+    /// view of the split values: over the value set there, when
+    /// `droppable` accepts it (as [`Self::split_field_mut`]), else as the
+    /// next value (as [`Self::split_append`]). The write barrier for the
+    /// value runs first. `Err` hands the value back, touching nothing,
+    /// with `true` when the old value was refused (the caller declines
+    /// the whole store) and `false` when the split layout can't take it.
+    ///
+    /// # Safety
+    ///
+    /// As [`crate::sync::GilCell::peek_mut`].
+    #[inline(always)]
+    pub unsafe fn split_store_at(
+        &self,
+        i: usize,
+        value: Object,
+        droppable: impl FnOnce(&Object) -> bool,
+    ) -> Result<(), (Object, bool)> {
+        let cls = self.cls_raw();
+        let Some(keys) = cls.shared_keys.get() else {
+            return Err((value, false));
+        };
+        if !value.is_gc_atomic() && self.deferred.get() {
+            self.ensure_gc_tracked();
+        }
+        // SAFETY: forwarded contract.
+        let Some(split) = (unsafe { self.dict.split_peek_mut() }) else {
+            return Err((value, false));
+        };
+        if let Some(slot) = split.get_over_mut(keys, i) {
+            if !droppable(slot) {
+                return Err((value, true));
+            }
+            drop(std::mem::replace(slot, value));
+            return Ok(());
+        }
+        if self.c_body.get() != 0 || crate::gil::free_threading_enabled() {
+            return Err((value, false));
+        }
+        split
+            .append_over(keys, || cls.shared_keys.share(), i, value)
+            .map_err(|v| (v, false))
+    }
+
     /// The split length and whether position `i` of the class's shared
     /// names is ready for a store: an overwrite of a set value (which
     /// `droppable` must accept), or the next of a run of in-order appends
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 5354c484..80b55e98 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -22387,24 +22387,21 @@ impl Interpreter {
                     return true;
                 }
             } else if cls.attr_version.get() == ver && !crate::capi_watchers::dicts_active() {
-                // SAFETY: a store between two instructions (see `peek_mut`).
-                if let Some(slot) =
-                    unsafe { inst.split_field_mut(idx as usize, value.is_gc_atomic()) }
-                {
-                    if Self::core_droppable(slot) {
-                        // SAFETY: as the indexed store below.
-                        drop(std::mem::replace(slot, unsafe { std::ptr::read(value) }));
-                        return true;
-                    }
-                    return false;
-                }
-                // The constructor shape: the attribute is the instance's
-                // next one.
-                // SAFETY: as above; the value moves out of the caller's
-                // stack slot, and a declined append hands the bits back.
-                match unsafe { inst.split_append(idx as usize, std::ptr::read(value)) } {
+                // Over the attribute's value, or (the constructor shape) as
+                // the instance's next one.
+                // SAFETY: a store between two instructions (see `peek_mut`);
+                // the value moves out of the caller's stack slot, and a
+                // declined store hands the bits back.
+                match unsafe {
+                    inst.split_store_at(idx as usize, std::ptr::read(value), Self::core_droppable)
+                } {
                     Ok(()) => return true,
-                    Err(v) => std::mem::forget(v),
+                    Err((v, refused)) => {
+                        std::mem::forget(v);
+                        if refused {
+                            return false;
+                        }
+                    }
                 }
             }
         }

From f464e429b2668bfd80d7d5dd448578ea8a7fc0d8 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 12:59:09 -0700
Subject: [PATCH 086/173] perf: count laid-out slot reads in the pure-leaf
 coverage test

The frameless evaluator's slot reads now mostly hit the field shortcut
at the top of `leaf_cached_instance_field`, which the unit test's path
counter didn't see, so `pure_slot_fields_preserve_lookup_and_lifetimes`
found too few hits. The shortcut's laid-out member reads count as the
inline-cache reads did, through one shared helper.
---
 crates/weavepy-vm/src/lib.rs | 38 ++++++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 80b55e98..d41ee37b 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -20036,6 +20036,13 @@ impl Interpreter {
         use weavepy_compiler::InlineCache as IC;
         // SAFETY: forwarded contract.
         if let Some(v) = unsafe { field_slot_hit(ext, cache_pc as usize, inst) } {
+            #[cfg(test)]
+            if ext
+                .field_slot(cache_pc as usize)
+                .is_some_and(|s| s.get().1 & SLOT_FIELD != 0)
+            {
+                note_pure_slot_field_read(code);
+            }
             return Some(v);
         }
         let names: &[Object] = &ext.name_objs;
@@ -20069,18 +20076,7 @@ impl Interpreter {
             // Slot order can vary by instance or after deletion.
             let value = indexed.or_else(|| slots.get(code.names.get(name_idx as usize)?))?;
             #[cfg(test)]
-            PURE_SLOT_FIELD_READS.with(|hits| {
-                let mut counts = hits.get();
-                let shape = code_vm_ext(code)
-                    .map(|ext| ext.pure_leaf.load(std::sync::atomic::Ordering::Relaxed));
-                let index = match shape {
-                    Some(6) => 0,
-                    Some(7) => 1,
-                    _ => 2,
-                };
-                counts[index] += 1;
-                hits.set(counts);
-            });
+            note_pure_slot_field_read(code);
             Some(value)
         }
     }
@@ -72854,6 +72850,24 @@ unsafe fn field_slot_hit<'a>(
     unsafe { inst.split_field(idx as usize) }
 }
 
+/// Count a frameless slot read by the shape of the leaf `code` reading it
+/// (the unit tests' path coverage).
+#[cfg(test)]
+fn note_pure_slot_field_read(code: &CodeObject) {
+    PURE_SLOT_FIELD_READS.with(|hits| {
+        let mut counts = hits.get();
+        let shape =
+            code_vm_ext(code).map(|ext| ext.pure_leaf.load(std::sync::atomic::Ordering::Relaxed));
+        let index = match shape {
+            Some(6) => 0,
+            Some(7) => 1,
+            _ => 2,
+        };
+        counts[index] += 1;
+        hits.set(counts);
+    });
+}
+
 /// A [`FieldSlot`] position naming a `__slots__` member's place in its
 /// class's slot layout (see [`crate::types::PyInstance::laid_out_slot`])
 /// rather than a split field's: CPython's member descriptor offset. No

From 802abcf61e720e0c7cef9e70d45fd33a6249164e Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 13:37:49 -0700
Subject: [PATCH 087/173] perf: lay a store-only constructor's values down in
 one pass

A store-only `__init__` appended each attribute to the fresh instance
through the full split append (the class's names, the write barrier,
the native-body and free-threading checks, a view of the values) once
per store. The values are now computed first, the barrier runs once
for all of them, and they go into the block reserved for the class's
names in one pass; anything the block can't take that way (a recycled
instance's smaller block, or its published `__dict__`) takes the
per-store path as before.

20-attribute constructor: 4067 -> 3194 instructions (CPython ~3080).
---
 crates/weavepy-vm/src/inst_dict.rs     | 40 +++++++++++++++++++
 crates/weavepy-vm/src/lib.rs           | 54 +++++++++++++++++++++-----
 tests/regrtest/test_wide_store_init.py | 35 +++++++++++++++++
 3 files changed, 119 insertions(+), 10 deletions(-)

diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index dbda5e07..d15e4760 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -612,6 +612,46 @@ impl SplitValues {
         self.block = Some(h);
     }
 
+    /// Lay `n` values down as the first `n` of values over `keys` (a
+    /// fresh instance's constructor stores, in its names' order), taking
+    /// value `k` from `value(k)` once each; a block without names adopts
+    /// them through `share`. `false` when the block can't take them all
+    /// at once (no block, values already set, too little room or other
+    /// names), touching nothing and calling `value` never.
+    pub(crate) fn fill_fresh(
+        &mut self,
+        keys: &SharedKeys,
+        share: impl FnOnce() -> Rc<SharedKeys>,
+        n: usize,
+        mut value: impl FnMut(usize) -> Object,
+    ) -> bool {
+        if n > keys.len() {
+            return false;
+        }
+        let Some(b) = self.block else {
+            return false;
+        };
+        let h = b.as_ptr();
+        // SAFETY: the block is live while owned; its first `n` slots are
+        // inside the capacity (checked) and unset (the length is zero).
+        unsafe {
+            if (*h).len != 0 || ((*h).cap as usize) < n {
+                return false;
+            }
+            if (*h).keys.is_null() {
+                (*h).keys = Rc::into_raw(share());
+            } else if !std::ptr::eq((*h).keys, keys) {
+                return false;
+            }
+            let values = Self::values_ptr(h);
+            for k in 0..n {
+                values.add(k).write(value(k));
+            }
+            (*h).len = n as u32;
+        }
+        true
+    }
+
     /// Whether [`Self::append_over`] of position `i` of `keys` succeeds
     /// once the positions from the current length up to `i` have been
     /// appended first (the next of a run of in-order appends).
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index d41ee37b..d963f898 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -17386,6 +17386,9 @@ impl Interpreter {
         let mut left = [0u8; 8];
         left[..shape.uses.len()].copy_from_slice(&shape.uses);
         let defaults = &init.defaults[init.defaults.len() - missing..];
+        // A split instance's values, all computed, go down in one pass.
+        let mut split_vals =
+            [const { std::mem::MaybeUninit::<Object>::uninit() }; crate::inst_dict::SHARED_KEYS_CAP];
         for (k, &(_, src)) in shape.stores.iter().enumerate() {
             let value = match src {
                 StoreSrc::Param(0) => inst.clone(),
@@ -17427,16 +17430,47 @@ impl Interpreter {
                 }
                 continue;
             }
-            // SAFETY: a store with nothing else running; the positions are
-            // the instance's next ones (proved above).
-            if let Err(value) = unsafe { i.split_append(k, value) } {
-                // (Unreachable for a fresh instance over its class's
-                // names; the plain store keeps it correct regardless.)
-                let at = code.instructions[shape.stores[k].0 as usize].arg;
-                if let Some(name) = code_name_obj(code, at) {
-                    i.dict_shared()
-                        .borrow_mut()
-                        .insert(DictKey(name.clone()), value);
+            split_vals[k].write(value);
+        }
+        if let Some(keys) = keys {
+            let n = shape.stores.len();
+            // SAFETY: the first `n` entries were written above; each is
+            // read exactly once, by the fill or by the stores after it.
+            let vals = unsafe {
+                std::slice::from_raw_parts_mut(split_vals.as_mut_ptr().cast::<Object>(), n)
+            };
+            // The write barrier, once, before the values are viewed.
+            if i.deferred.get() && !vals.iter().all(Object::is_gc_atomic) {
+                i.ensure_gc_tracked();
+            }
+            // (A recycled instance may carry a published, empty `__dict__`,
+            // which then holds its attributes instead.)
+            // SAFETY: the instance is fresh and unshared.
+            let filled = i.c_body.get() == 0
+                && !crate::gil::free_threading_enabled()
+                && unsafe { i.dict.split_peek_mut() }.is_some_and(|split| {
+                    let at = vals.as_ptr();
+                    // SAFETY: as above.
+                    split.fill_fresh(keys, || ty.shared_keys.share(), n, |k| unsafe {
+                        at.add(k).read()
+                    })
+                });
+            if !filled {
+                for k in 0..n {
+                    // SAFETY: as above.
+                    let value = unsafe { std::ptr::read(&vals[k]) };
+                    // SAFETY: a store with nothing else running; the
+                    // positions are the instance's next ones (proved above).
+                    if let Err(value) = unsafe { i.split_append(k, value) } {
+                        // (Unreachable for a fresh instance over its class's
+                        // names; the plain store keeps it correct regardless.)
+                        let at = code.instructions[shape.stores[k].0 as usize].arg;
+                        if let Some(name) = code_name_obj(code, at) {
+                            i.dict_shared()
+                                .borrow_mut()
+                                .insert(DictKey(name.clone()), value);
+                        }
+                    }
                 }
             }
         }
diff --git a/tests/regrtest/test_wide_store_init.py b/tests/regrtest/test_wide_store_init.py
index d6899dbc..76b012ab 100644
--- a/tests/regrtest/test_wide_store_init.py
+++ b/tests/regrtest/test_wide_store_init.py
@@ -50,6 +50,41 @@ def test_widths(self):
                 with self.subTest(n=n, slots=slots):
                     self.check(n, slots)
 
+    def test_tracked_values_and_cycles(self):
+        import gc
+        import weakref
+
+        class Node:
+            def __init__(self, other, tag, items):
+                self.other = other
+                self.tag = tag
+                self.items = items
+
+        refs = []
+        for i in range(500):
+            a = Node(None, i, [i])
+            b = Node(a, i, [a])
+            a.other = b
+            refs.append(weakref.ref(a))
+            del a, b
+        gc.collect()
+        self.assertTrue(all(r() is None for r in refs))
+        n = Node(None, 1, [])
+        m = Node(n, 2, [n])
+        self.assertIs(m.other, n)
+        self.assertIs(m.items[0], n)
+
+    def test_reused_instance_with_published_dict(self):
+        cls, nparams = make_class(5)
+        for i in range(2000):
+            a = cls(*range(nparams))
+            # A published `__dict__`, kept (emptied) by a pooled instance.
+            self.assertEqual(vars(a), expected(5))
+            del a
+            b = cls(*range(nparams))
+            self.assertEqual(vars(b), expected(5))
+            self.assertEqual(b.f0, 0)
+
     def test_class_change_between_calls(self):
         cls, nparams = make_class(20)
         for _ in range(300):

From e72928c63405e4a9d6097f065792bd20fba86a73 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 15:14:31 -0700
Subject: [PATCH 088/173] perf: drop discarded instances plainly in compiled
 pops again

Pooling the instances compiled code's `POP_TOP` discards paid for
itself only in a microbenchmark: under callgrind it cost go 0.5% and
raytrace 0.2% (the reset and pool push per death outweigh the reuse),
so the pop drops them plainly again. `PyInstance::recycle` now also
checks the pool has room before resetting anything, and the field
shortcut's store tests the class version once before choosing between
a slot member and a split position.
---
 crates/weavepy-vm/src/frame_jit.rs | 11 -----------
 crates/weavepy-vm/src/lib.rs       |  7 +++++--
 crates/weavepy-vm/src/types.rs     |  8 ++++++++
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 37d04e60..f9bd8b00 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1146,17 +1146,6 @@ unsafe extern "C" fn h_pop(slot: *mut Object) -> u32 {
         if !droppable(&*slot) {
             return 1;
         }
-        // A plain instance dying here goes to the instance pool, as the
-        // core loop's release sends it (`C(...)` as a statement).
-        if let Object::Instance(i) = &*slot {
-            if Rc::strong_count(i) == 1 && i.dies_by_plain_drop() {
-                let Object::Instance(i) = slot.read() else {
-                    unreachable!("matched above")
-                };
-                crate::types::PyInstance::try_recycle(i);
-                return 0;
-            }
-        }
         crate::drop_hot(slot.read());
     }
     0
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index d963f898..fec8bf26 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -22402,7 +22402,10 @@ impl Interpreter {
         let cls = inst.cls_raw();
         let ext = code_vm_ext(code);
         if let Some((ver, idx)) = ext.and_then(|e| e.field_slot(attr_pc)).map(FieldSlot::get) {
-            if idx & SLOT_FIELD != 0 && cls.attr_version.get() == ver {
+            if cls.attr_version.get() != ver {
+                // Another class's (or version's) shortcut: the site's
+                // cache below decides.
+            } else if idx & SLOT_FIELD != 0 {
                 // A laid-out `__slots__` member, set or not: the member
                 // descriptor's store, in place.
                 // SAFETY: a store between two instructions (see `peek_mut`).
@@ -22416,7 +22419,7 @@ impl Interpreter {
                     drop(std::mem::replace(slot, unsafe { std::ptr::read(value) }));
                     return true;
                 }
-            } else if cls.attr_version.get() == ver && !crate::capi_watchers::dicts_active() {
+            } else if !crate::capi_watchers::dicts_active() {
                 // Over the attribute's value, or (the constructor shape) as
                 // the instance's next one.
                 // SAFETY: a store between two instructions (see `peek_mut`);
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 442c7de9..55f0fd5c 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -3660,6 +3660,14 @@ impl PyInstance {
         if unsafe { &*inst.class.as_ptr() }.native_kind.get() != 0 {
             return crate::stdlib::datetime_native::recycle(inst);
         }
+        // A full pool (the program discards more instances than it makes
+        // through the pool) takes nothing: skip the reset.
+        let room = INSTANCE_POOL
+            .try_with(|p| p.try_borrow().is_ok_and(|p| p.len() < INSTANCE_POOL_CAP))
+            .unwrap_or(false);
+        if !room {
+            return Err(inst);
+        }
         let Some(m) = Rc::get_mut(&mut inst) else {
             return Err(inst);
         };

From 79c0debbd667130f7668726c608790112d15d29d Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 13:01:42 -0700
Subject: [PATCH 089/173] perf: lay out exception slots from birth and slim
 handled-exception records

An exception instance kept its pseudo-slots in a per-key table that grew
as a raise went on: `args` first, then a traceback store that probed the
table, built an interned key and pushed (reallocating for a
`StopIteration`, whose `value` came second), then the chaining links.
Exceptions are now born over a shared slot layout (`args`, the family's
own fields, then `__traceback__`, `__context__`, `__cause__` and
`__suppress_context__`), so a raise fills its slots in place. A field the
layout doesn't name converts to the per-key form, as a `__slots__`
member's store into a laid-out instance always has. Slot names of four to
sixteen bytes compare as two overlapping words.

`PyException` no longer carries a Rust-side list of traceback entries
beside the instance's `__traceback__` chain; a flag records that the
exception unwound through a frame (what the list's emptiness answered).
The CLI's fallback report and the default unraisable hook read the chain,
outermost frame first.

The handled-exception stack (`sys.exc_info()`, a suspended generator's
saved handler state) holds the instances themselves instead of bare
`PyException` records, and `PUSH_EXC_INFO` takes one borrow of it.

Per-iteration instructions (probes): `raise StopIteration` caught 4355 ->
3718, `raise ValueError(i)` caught 4344 -> 4088, a call raising
`KeyError` caught 8889 -> 8427, `next(gen)` to exhaustion caught 7668 ->
7407, `@contextmanager` with 23311 -> 22926. Suite (instructions per
call vs. the branch base): exceptions 0.972, context_managers 0.988,
geomean 1.003 (logging_mix 1.028 and sorting 1.020 move with code layout;
under callgrind logging_mix is 0.990).
---
 crates/weavepy-vm/src/builtin_types.rs        |  61 ++--
 crates/weavepy-vm/src/error.rs                |  61 ++--
 crates/weavepy-vm/src/lib.rs                  | 130 +++++----
 .../weavepy-vm/src/stdlib/faulthandler_mod.rs |   6 +-
 .../weavepy-vm/src/stdlib/greenlet_native.rs  |   2 +-
 crates/weavepy-vm/src/stdlib/sys.rs           |  12 +-
 crates/weavepy-vm/src/types.rs                | 126 +++++++-
 crates/weavepy-vm/src/vm_singletons.rs        |   2 +-
 crates/weavepy/src/lib.rs                     |   9 +-
 tests/regrtest/test_exception_slot_layout.py  | 276 ++++++++++++++++++
 10 files changed, 537 insertions(+), 148 deletions(-)
 create mode 100644 tests/regrtest/test_exception_slot_layout.py

diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs
index fe2f2851..685e3afe 100644
--- a/crates/weavepy-vm/src/builtin_types.rs
+++ b/crates/weavepy-vm/src/builtin_types.rs
@@ -4894,45 +4894,50 @@ pub(crate) fn exception_from_parts_with<const N: usize>(
         Some(a) => Object::new_tuple_array([a.clone()]),
     };
     inst.note_slot_store(&args);
-    let mut entries: Vec<(crate::object::DictKey, Object)> = Vec::with_capacity(4);
     // PEP 380: `StopIteration.value` is always present (CPython sets it
     // in `StopIteration.__init__`, defaulting to None). A Rust-raised
     // bare `StopIteration` must answer `.value` too: asyncio's
     // `Task.__step` reads `exc.value` on every coroutine return, and a
     // missing attribute leaves the task wedged (gh: shutdown_asyncgens).
-    if families & crate::EXC_FAM_STOP_ITERATION != 0 {
-        let value = arg.clone().unwrap_or(Object::None);
-        inst.note_slot_store(&value);
-        entries.push((slot_key("value"), value));
-    }
-    entries.push((slot_key("args"), args));
+    let value = (families & crate::EXC_FAM_STOP_ITERATION != 0)
+        .then(|| arg.clone().unwrap_or(Object::None));
     // The BaseException pseudo-slots (`__context__`/`__cause__`/
     // `__suppress_context__`/`__traceback__`) and OSError's named fields
     // (`errno`/`strerror`/…) are *not* seeded per-instance: their class
     // slot descriptors answer the CPython getset defaults while unset,
     // and a subclass's own class attribute (`class Err(OSError): errno =
     // EINVAL`, raised bare via `raise Err`) stays visible.
-    if families & (crate::EXC_FAM_IMPORT_ERROR | crate::EXC_FAM_SYNTAX_ERROR) != 0 {
-        // ImportError/ModuleNotFoundError expose `msg` (the message
-        // string). CPython always defines the slot; a Rust-raised
-        // ImportError must answer `.msg` so consumers (e.g. numpy's
-        // `_core/__init__` error handling, which reads `exc.msg`) don't
-        // AttributeError. `name`/`path`/`name_from` are intentionally
-        // *not* pre-set here so the import machinery's
-        // `set_exception_attr` (which skips already-present slots) can
-        // still populate the real module name. SyntaxError gets `msg`
-        // from `args[0]` too; the location payload reads `None` off the
-        // class descriptors until `error::syntax_error_located` fills
-        // real values.
-        let msg = arg.unwrap_or_else(|| Object::from_static(""));
-        inst.note_slot_store(&msg);
-        entries.push((slot_key("msg"), msg));
-    }
-    for (name, value) in extra {
-        inst.note_slot_store(&value);
-        entries.push((slot_key(name), value));
-    }
-    *inst.slots.borrow_mut() = SlotStorage::from_entries(entries);
+    //
+    // ImportError/ModuleNotFoundError expose `msg` (the message
+    // string). CPython always defines the slot; a Rust-raised
+    // ImportError must answer `.msg` so consumers (e.g. numpy's
+    // `_core/__init__` error handling, which reads `exc.msg`) don't
+    // AttributeError. `name`/`path`/`name_from` are intentionally
+    // *not* pre-set here so the import machinery's
+    // `set_exception_attr` (which skips already-present slots) can
+    // still populate the real module name. SyntaxError gets `msg`
+    // from `args[0]` too; the location payload reads `None` off the
+    // class descriptors until `error::syntax_error_located` fills
+    // real values.
+    let msg = (families & (crate::EXC_FAM_IMPORT_ERROR | crate::EXC_FAM_SYNTAX_ERROR) != 0)
+        .then(|| arg.unwrap_or_else(|| Object::from_static("")));
+    for v in value.iter().chain(&msg).chain(extra.iter().map(|(_, v)| v)) {
+        inst.note_slot_store(v);
+    }
+    let slots = match (value, msg) {
+        (None, None) => SlotStorage::exception(args, extra),
+        (Some(value), None) if N == 0 => SlotStorage::exception(args, [("value", value)]),
+        (None, Some(msg)) if N == 0 => SlotStorage::exception(args, [("msg", msg)]),
+        (value, msg) => {
+            let mut entries = Vec::with_capacity(N + 3);
+            entries.push((slot_key("args"), args));
+            entries.extend(value.map(|v| (slot_key("value"), v)));
+            entries.extend(msg.map(|m| (slot_key("msg"), m)));
+            entries.extend(extra.into_iter().map(|(k, v)| (slot_key(k), v)));
+            SlotStorage::from_entries(entries)
+        }
+    };
+    *inst.slots.borrow_mut() = slots;
     Object::Instance(Rc::new(inst))
 }
 
diff --git a/crates/weavepy-vm/src/error.rs b/crates/weavepy-vm/src/error.rs
index 3b2e1664..33695bad 100644
--- a/crates/weavepy-vm/src/error.rs
+++ b/crates/weavepy-vm/src/error.rs
@@ -37,44 +37,16 @@ impl PartialEq for TracebackEntry {
     }
 }
 
-/// An exception's [`TracebackEntry`] list, innermost first, with the
-/// first entry held inline: most exceptions are caught in the frame that
-/// raised them or the next one, and a raise then allocates nothing here.
-#[derive(Debug, Clone, Default, PartialEq)]
-pub struct TracebackEntries {
-    first: Option<TracebackEntry>,
-    rest: Vec<TracebackEntry>,
-}
-
-impl TracebackEntries {
-    pub fn push(&mut self, entry: TracebackEntry) {
-        if self.first.is_none() {
-            self.first = Some(entry);
-        } else {
-            self.rest.push(entry);
-        }
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.first.is_none()
-    }
-
-    pub fn len(&self) -> usize {
-        usize::from(self.first.is_some()) + self.rest.len()
-    }
-
-    /// The entries in push order (innermost frame first).
-    pub fn iter(&self) -> impl DoubleEndedIterator<Item = &TracebackEntry> {
-        self.first.iter().chain(self.rest.iter())
-    }
-}
-
 /// A Python-visible exception. The wrapped [`Object`] is always an
 /// `Object::Instance` whose class's MRO contains `BaseException`.
 #[derive(Debug, Clone)]
 pub struct PyException {
     pub instance: Object,
-    pub traceback: TracebackEntries,
+    /// Whether the exception has unwound through a frame (which added
+    /// that frame's entry to the instance's `__traceback__` chain): a
+    /// fresh Rust-raised error has not. The entries themselves live only
+    /// in the chain (see [`Self::traceback_entries`]).
+    pub traced: bool,
     /// Implicit chaining context (`raise X` inside `except Y:` records
     /// `Y` as `__context__`). Stored separately from `instance.__dict__`
     /// so re-raises through pure Rust paths keep the link intact.
@@ -97,7 +69,7 @@ impl PyException {
     pub fn new(instance: Object) -> Self {
         Self {
             instance,
-            traceback: TracebackEntries::default(),
+            traced: false,
             context: None,
             cause: None,
             suppress_tb_once: false,
@@ -127,8 +99,25 @@ impl PyException {
         crate::builtin_types::exception_message(&self.instance).unwrap_or_default()
     }
 
-    pub fn push_traceback(&mut self, entry: TracebackEntry) {
-        self.traceback.push(entry);
+    /// The instance's `__traceback__` chain as entries, outermost frame
+    /// first ("most recent call last").
+    pub fn traceback_entries(&self) -> Vec<TracebackEntry> {
+        let mut entries = Vec::new();
+        let Object::Instance(inst) = &self.instance else {
+            return entries;
+        };
+        let mut cur = match inst.slot_get("__traceback__") {
+            Some(Object::Traceback(tb)) => Some(tb),
+            _ => None,
+        };
+        while let Some(node) = cur {
+            entries.push(TracebackEntry {
+                code: node.code(),
+                lineno: node.lineno,
+            });
+            cur = node.next.borrow().clone();
+        }
+        entries
     }
 
     /// PEP 678: append a string note to the wrapped instance's
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index fec8bf26..754d64fb 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -102,7 +102,7 @@ use crate::error::{
     attribute_error, import_error, index_error, key_error, key_error_object,
     module_not_found_error, name_error, overflow_error, recursion_error, runtime_error,
     stop_async_iteration, stop_iteration, stop_iteration_with, type_error, value_error,
-    zero_division_error, TracebackEntry,
+    zero_division_error,
 };
 pub use crate::error::{PyException, RuntimeError};
 pub use crate::import::ModuleCache;
@@ -272,7 +272,7 @@ struct FrameExc {
     /// are peeled back off so they don't leak into the *resumer's*
     /// `sys.exc_info()`. Always empty for ordinary frames (they never
     /// suspend).
-    saved: Vec<PyException>,
+    saved: Vec<Object>,
     /// pc of the instruction whose exception is being handled by a
     /// *cleanup* handler (`push_lasti` exception-table entries: `with`
     /// exits, except-variable unbind blocks). CPython pushes this on
@@ -327,12 +327,12 @@ impl Frame {
     }
 
     /// The innermost [`FrameExc::saved`] entry.
-    fn saved_exc_info_last(&self) -> Option<&PyException> {
+    fn saved_exc_info_last(&self) -> Option<&Object> {
         self.exc.as_ref()?.saved.last()
     }
 
     /// Take the [`FrameExc::saved`] entries.
-    fn take_saved_exc_info(&mut self) -> Vec<PyException> {
+    fn take_saved_exc_info(&mut self) -> Vec<Object> {
         self.exc
             .as_mut()
             .map(|e| std::mem::take(&mut e.saved))
@@ -340,7 +340,7 @@ impl Frame {
     }
 
     /// Set the [`FrameExc::saved`] entries.
-    fn set_saved_exc_info(&mut self, saved: Vec<PyException>) {
+    fn set_saved_exc_info(&mut self, saved: Vec<Object>) {
         if saved.is_empty() && self.exc.is_none() {
             return;
         }
@@ -867,7 +867,7 @@ pub struct Interpreter {
     /// Stack of currently-handled exceptions across all frames. The
     /// top is what `sys.exc_info()` returns. Pushed by
     /// `PUSH_EXC_INFO`; popped by `POP_EXCEPT`.
-    pub(crate) exc_info_stack: Rc<RefCell<Vec<PyException>>>,
+    pub(crate) exc_info_stack: Rc<RefCell<Vec<Object>>>,
     /// User-installable hook called when an exception escapes the
     /// top-level frame. Defaults to a Rust builtin that prints the
     /// canonical CPython-style traceback to `sys.stderr`.
@@ -3315,7 +3315,7 @@ impl Interpreter {
                 (
                     inst,
                     ty,
-                    pyexc.traceback.iter().cloned().collect::<Vec<_>>(),
+                    pyexc.traceback_entries(),
                 )
             }
             other => {
@@ -17684,11 +17684,12 @@ impl Interpreter {
                 }
                 // SAFETY: as above.
                 let instance = unsafe { &*base.add(len - 1) }.clone();
-                let prev = self
-                    .exc_info_stack
-                    .borrow()
-                    .last()
-                    .map_or(Object::None, |pe| pe.instance.clone());
+                let prev = {
+                    let mut handled = self.exc_info_stack.borrow_mut();
+                    let prev = handled.last().cloned().unwrap_or(Object::None);
+                    handled.push(instance);
+                    prev
+                };
                 // SAFETY: the exception moves up one slot and the previous
                 // one takes its place (`len < cap`).
                 unsafe {
@@ -17696,9 +17697,6 @@ impl Interpreter {
                     base.add(len - 1).write(prev);
                     base.add(len).write(top);
                 }
-                self.exc_info_stack
-                    .borrow_mut()
-                    .push(PyException::new(instance));
                 Some(len + 1)
             }
             OpCode::CheckExcMatch => {
@@ -17725,8 +17723,8 @@ impl Interpreter {
                 let top = self.exc_info_stack.borrow_mut().pop();
                 // See the full handler: the handled exception dies here
                 // unless something else holds it.
-                if let Some(pe) = top {
-                    self.release(pe.instance);
+                if let Some(top) = top {
+                    self.release(top);
                 }
                 self.release(prev);
                 Some(len - 1)
@@ -24758,14 +24756,14 @@ impl Interpreter {
                         break;
                     }
                     let exc = stack.pop().expect("checked above");
-                    let prev = self
-                        .exc_info_stack
-                        .borrow()
-                        .last()
-                        .map_or(Object::None, |pe| pe.instance.clone());
+                    let prev = {
+                        let mut handled = self.exc_info_stack.borrow_mut();
+                        let prev = handled.last().cloned().unwrap_or(Object::None);
+                        handled.push(exc.clone());
+                        prev
+                    };
                     stack.push(prev);
-                    stack.push(exc.clone());
-                    self.exc_info_stack.borrow_mut().push(PyException::new(exc));
+                    stack.push(exc);
                     last = pc;
                     pc += 1;
                 }
@@ -24789,8 +24787,8 @@ impl Interpreter {
                     // here unless something else holds it.
                     last = pc;
                     pc += 1;
-                    if let Some(pe) = top {
-                        self.release(pe.instance);
+                    if let Some(top) = top {
+                        self.release(top);
                     }
                     self.release(prev);
                 }
@@ -30841,12 +30839,13 @@ impl Interpreter {
                         // (`sys.exc_info()`), not frame-local: a helper
                         // called from inside an `except:` block can
                         // re-raise the caller's exception.
-                        let mut top = self
+                        let top = self
                             .exc_info_stack
                             .borrow()
                             .last()
                             .cloned()
                             .ok_or_else(|| runtime_error("No active exception to re-raise"))?;
+                        let mut top = PyException::new(top);
                         top.suppress_tb_once = true;
                         top.context_settled = true;
                         return Err(RuntimeError::PyException(top));
@@ -30999,9 +30998,8 @@ impl Interpreter {
                 // `raise`'s `__context__` see the match, not the whole
                 // group, with no per-clause PUSH_EXC_INFO on the wire.
                 if !matches!(matched, Object::None) {
-                    let pe = PyException::new(matched.clone());
                     if let Some(top) = self.exc_info_stack.borrow_mut().last_mut() {
-                        *top = pe;
+                        *top = matched.clone();
                     }
                 }
                 frame.push(rest);
@@ -31019,12 +31017,12 @@ impl Interpreter {
                     .exc_info_stack
                     .borrow()
                     .last()
-                    .map(|pe| pe.instance.clone())
+                    .cloned()
                     .unwrap_or(Object::None);
                 frame.push(prev);
                 frame.push(exc.clone());
                 if let Object::Instance(_) = &exc {
-                    self.exc_info_stack.borrow_mut().push(PyException::new(exc));
+                    self.exc_info_stack.borrow_mut().push(exc);
                 }
             }
             OpCode::PopExcept => {
@@ -31039,8 +31037,8 @@ impl Interpreter {
                 // `except` block (the implicit `del` of the bound name plus
                 // the per-frame exc-state pop): it dies here unless something
                 // else holds it.
-                if let Some(pe) = popped {
-                    self.release(pe.instance);
+                if let Some(popped) = popped {
+                    self.release(popped);
                 }
             }
             OpCode::PrepReraiseStar => {
@@ -31489,7 +31487,7 @@ impl Interpreter {
                         // Keep the in-flight traceback so the RuntimeError
                         // points at the offending frame; chain the original
                         // as cause + context (PEP 479 wording).
-                        new_exc.traceback = orig.traceback.clone();
+                        new_exc.traced = orig.traced;
                         new_exc.cause = Some(Box::new(orig.clone()));
                         new_exc.context = Some(Box::new(orig));
                         new_exc.context_settled = true;
@@ -31878,7 +31876,7 @@ impl Interpreter {
         let fresh = !exc.context_settled
             && exc.context.is_none()
             && exc.cause.is_none()
-            && exc.traceback.is_empty()
+            && !exc.traced
             && !exc.suppress_tb_once;
         // Nothing is being handled: there is no context to chain (the
         // decision `attach_implicit_context` would settle the same way),
@@ -31892,7 +31890,7 @@ impl Interpreter {
             if exc.context.is_some() {
                 Self::sync_exc_attrs(exc);
             }
-        } else if exc.traceback.is_empty() && (exc.cause.is_some() || exc.context.is_some()) {
+        } else if !exc.traced && (exc.cause.is_some() || exc.context.is_some()) {
             // A fresh Rust-raised error that already carries an
             // explicit `cause`/`context` (e.g. `_io` chaining a
             // misbehaving raw `readinto`'s `TypeError` into an
@@ -32145,15 +32143,10 @@ impl Interpreter {
     }
 
     /// Push one frame's worth of `tb_*` info onto the exception's
-    /// traceback chain — both the legacy `Vec<TracebackEntry>` (used
-    /// for the cheap `RuntimeError` Display impl) and the new
-    /// `PyTraceback` chain stored on the instance dict so Python code
-    /// can walk `exc.__traceback__`.
+    /// traceback: the `PyTraceback` chain stored on the instance so
+    /// Python code can walk `exc.__traceback__`.
     fn append_traceback(&self, exc: &mut PyException, frame: &mut Frame, lasti: u32, lineno: u32) {
-        exc.push_traceback(TracebackEntry {
-            code: frame.code.clone(),
-            lineno,
-        });
+        exc.traced = true;
         // The Python-visible frame for this entry must be *this* frame's
         // snapshot. Generator-family frames cache theirs (stable identity
         // across resumes) but are not necessarily on `frame_stack` when an
@@ -32235,12 +32228,17 @@ impl Interpreter {
             let new_tb = Object::Traceback(new_tb);
             inst.note_slot_store(&new_tb);
             let mut slots = inst.slots.borrow_mut();
-            match slots.get_mut("__traceback__") {
+            match slots.place_mut("__traceback__") {
                 Some(slot) => {
                     let prev = std::mem::replace(slot, new_tb);
-                    if let (Object::Traceback(prev_tb), Object::Traceback(head)) = (&prev, &*slot) {
-                        *head.next.borrow_mut() = Some(prev_tb.clone());
-                    }
+                    // The chain so far moves under the new head.
+                    let prev = match (prev, &*slot) {
+                        (Object::Traceback(prev_tb), Object::Traceback(head)) => {
+                            *head.next.borrow_mut() = Some(prev_tb);
+                            None
+                        }
+                        (prev, _) => Some(prev),
+                    };
                     drop(slots);
                     drop(prev);
                 }
@@ -32305,7 +32303,7 @@ impl Interpreter {
         let Some(ctx) = stack.last() else {
             return;
         };
-        let Object::Instance(ctx_inst) = &ctx.instance else {
+        let Object::Instance(ctx_inst) = ctx else {
             return;
         };
         let Object::Instance(exc_inst) = &exc.instance else {
@@ -32322,7 +32320,7 @@ impl Interpreter {
         // first so traceback walkers can't loop. CPython does the same
         // before assigning the new `__context__`.
         Self::break_implicit_context_cycle(ctx_inst, exc_inst);
-        exc.context = Some(Box::new(ctx.clone()));
+        exc.context = Some(Box::new(PyException::new(ctx.clone())));
     }
 
     /// Chain an exception injected by `gen.throw()` to whatever the
@@ -32336,7 +32334,7 @@ impl Interpreter {
         let Some(active) = frame.saved_exc_info_last() else {
             return;
         };
-        let Object::Instance(active_inst) = &active.instance else {
+        let Object::Instance(active_inst) = active else {
             return;
         };
         let Object::Instance(exc_inst) = &exc.instance else {
@@ -32346,7 +32344,7 @@ impl Interpreter {
             return;
         }
         Self::break_implicit_context_cycle(active_inst, exc_inst);
-        exc.context = Some(Box::new(active.clone()));
+        exc.context = Some(Box::new(PyException::new(active.clone())));
         // Mirror onto the instance dict so Python's `e.__context__` sees
         // it (the throw path doesn't go through `RAISE_VARARGS`, which is
         // where `sync_exc_attrs` normally runs).
@@ -41622,7 +41620,7 @@ impl Interpreter {
                         let RuntimeError::PyException(mut outer) = not_awaitable(&value) else {
                             unreachable!("not_awaitable builds a PyException");
                         };
-                        outer.traceback = inner.traceback.clone();
+                        outer.traced = inner.traced;
                         outer.cause = Some(Box::new(inner.clone()));
                         outer.context = Some(Box::new(inner));
                         Self::sync_exc_attrs(&outer);
@@ -43037,7 +43035,7 @@ impl Interpreter {
         let mut new_exc = PyException::new(rt_inst);
         // Keep the in-flight traceback so the RuntimeError points at the
         // offending frame; chain the original as cause + context.
-        new_exc.traceback = exc.traceback.clone();
+        new_exc.traced = exc.traced;
         new_exc.cause = Some(Box::new(exc.clone()));
         new_exc.context = Some(Box::new(exc));
         Self::sync_exc_attrs(&new_exc);
@@ -56039,9 +56037,7 @@ impl Interpreter {
             let inst = PyInstance::new(cls);
             let args_tuple = Object::new_tuple(args.to_vec());
             inst.note_slot_store(&args_tuple);
-            let mut entries = Vec::with_capacity(4);
-            entries.push((crate::types::slot_key("args"), args_tuple));
-            *inst.slots.borrow_mut() = crate::types::SlotStorage::from_entries(entries);
+            *inst.slots.borrow_mut() = crate::types::SlotStorage::exception(args_tuple, []);
             let obj = Object::Instance(Rc::new(inst));
             // See the GC note at the end.
             if args.iter().any(|x| !crate::gc_trace::is_atomic(x)) {
@@ -56069,13 +56065,27 @@ impl Interpreter {
         };
         // (No `message` mirror: CPython has none, and `str()` and the Rust
         // display derive the text from `args`.)
-        inst.slot_set("args", args_tuple);
         // PEP 380: `StopIteration.value` is the first constructor arg
         // (or None). Generator `return` goes through
         // `stop_iteration_with`, but user code constructs
         // `StopIteration(x)` directly and reads `.value` too.
-        if is_stop_iteration {
-            inst.slot_set("value", args.first().cloned().unwrap_or(Object::None));
+        let value = is_stop_iteration.then(|| args.first().cloned().unwrap_or(Object::None));
+        if inst.slots.borrow().is_empty_default() {
+            // Laid out as the common case's (see `SlotStorage::exception`).
+            inst.note_slot_store(&args_tuple);
+            let slots = match value {
+                Some(value) => {
+                    inst.note_slot_store(&value);
+                    crate::types::SlotStorage::exception(args_tuple, [("value", value)])
+                }
+                None => crate::types::SlotStorage::exception(args_tuple, []),
+            };
+            *inst.slots.borrow_mut() = slots;
+        } else {
+            inst.slot_set("args", args_tuple);
+            if let Some(value) = value {
+                inst.slot_set("value", value);
+            }
         }
         // `SystemExit.code`: None for no args, the lone argument for
         // one, the whole tuple otherwise (CPython `SystemExit_init`).
diff --git a/crates/weavepy-vm/src/stdlib/faulthandler_mod.rs b/crates/weavepy-vm/src/stdlib/faulthandler_mod.rs
index 45191fb9..d773b85d 100644
--- a/crates/weavepy-vm/src/stdlib/faulthandler_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/faulthandler_mod.rs
@@ -67,7 +67,7 @@ static WATCHDOG_GEN: AtomicU64 = AtomicU64::new(0);
 struct RegisteredThread {
     ident: u64,
     frame_stack: crate::object::FrameStack,
-    exc_info_stack: Rc<RefCell<Vec<crate::error::PyException>>>,
+    exc_info_stack: Rc<RefCell<Vec<Object>>>,
 }
 
 /// Registration order == thread creation order; CPython's
@@ -79,7 +79,7 @@ static THREADS: Mutex<Vec<RegisteredThread>> = Mutex::new(Vec::new());
 pub fn note_thread_start(
     ident: u64,
     frame_stack: crate::object::FrameStack,
-    exc_info_stack: Rc<RefCell<Vec<crate::error::PyException>>>,
+    exc_info_stack: Rc<RefCell<Vec<Object>>>,
 ) {
     let mut g = THREADS.lock().unwrap();
     if g.iter().any(|t| t.ident == ident) {
@@ -100,7 +100,7 @@ pub fn note_thread_start(
 pub fn thread_snapshots() -> Vec<(
     u64,
     crate::object::FrameStack,
-    Rc<RefCell<Vec<crate::error::PyException>>>,
+    Rc<RefCell<Vec<Object>>>,
 )> {
     THREADS
         .lock()
diff --git a/crates/weavepy-vm/src/stdlib/greenlet_native.rs b/crates/weavepy-vm/src/stdlib/greenlet_native.rs
index a6e2e996..1cea68e7 100644
--- a/crates/weavepy-vm/src/stdlib/greenlet_native.rs
+++ b/crates/weavepy-vm/src/stdlib/greenlet_native.rs
@@ -143,7 +143,7 @@ struct GreenletBody {
     /// whenever the greenlet is delivered to.
     frames: crate::object::FrameStack,
     /// This greenlet's own handled-exception stack.
-    exc_info: Rc<RefCell<Vec<PyException>>>,
+    exc_info: Rc<RefCell<Vec<Object>>>,
     /// `recursion::DEPTH` snapshot while parked.
     saved_depth: Cell<usize>,
     /// `gr_context` while not running (`None` = fresh implicit context;
diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs
index 049a5220..d0dd0c5d 100644
--- a/crates/weavepy-vm/src/stdlib/sys.rs
+++ b/crates/weavepy-vm/src/stdlib/sys.rs
@@ -34,7 +34,7 @@ pub const PY_VERSION: (i64, i64, i64) = (
 pub fn build_with_state(
     cache: &ModuleCache,
     frame_stack: crate::object::FrameStack,
-    exc_info_stack: Rc<RefCell<Vec<crate::error::PyException>>>,
+    exc_info_stack: Rc<RefCell<Vec<Object>>>,
     excepthook: Rc<RefCell<Object>>,
     unraisable_hook: Rc<RefCell<Object>>,
 ) -> Rc<PyModule> {
@@ -403,7 +403,7 @@ pub fn build_with_state(
                             let value = exc
                                 .borrow()
                                 .last()
-                                .map(|top| top.instance.clone())
+                                .cloned()
                                 .unwrap_or(Object::None);
                             d.insert(DictKey(Object::Int(ident as i64)), value);
                         }
@@ -1661,12 +1661,12 @@ fn sys_getframemodulename(
 /// being handled, or `None` if not in an `except`. Equivalent to
 /// `sys.exc_info()[1]`. The verbatim CPython `contextlib` relies on this.
 fn sys_exception(
-    exc_info_stack: &Rc<RefCell<Vec<crate::error::PyException>>>,
+    exc_info_stack: &Rc<RefCell<Vec<Object>>>,
 ) -> Result<Object, RuntimeError> {
     let stack = exc_info_stack.borrow();
     Ok(stack
         .last()
-        .map(|top| top.instance.clone())
+        .cloned()
         .or_else(capi_handled_exception_instance)
         .unwrap_or(Object::None))
 }
@@ -1682,12 +1682,12 @@ fn capi_handled_exception_instance() -> Option<Object> {
 }
 
 fn sys_exc_info(
-    exc_info_stack: &Rc<RefCell<Vec<crate::error::PyException>>>,
+    exc_info_stack: &Rc<RefCell<Vec<Object>>>,
 ) -> Result<Object, RuntimeError> {
     let stack = exc_info_stack.borrow();
     let inst = stack
         .last()
-        .map(|top| top.instance.clone())
+        .cloned()
         .or_else(capi_handled_exception_instance);
     if let Some(inst) = inst {
         let type_obj = match &inst {
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 55f0fd5c..347cffa8 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -2503,18 +2503,26 @@ fn slot_name_eq(stored: &str, name: &str) -> bool {
     if a.len() != b.len() {
         return false;
     }
-    if a.len() > 16 {
+    let n = a.len();
+    // (Two overlapping words cover every length from one word to two.)
+    let word = |s: &[u8], at: usize| u64::from_ne_bytes(s[at..at + 8].try_into().unwrap());
+    let half = |s: &[u8], at: usize| u32::from_ne_bytes(s[at..at + 4].try_into().unwrap());
+    match n {
         // Past a couple of words the library routine's vector loop wins.
-        return a == b;
-    }
-    let mut i = 0;
-    while i < a.len() {
-        if a[i] != b[i] {
-            return false;
+        17.. => a == b,
+        8..=16 => word(a, 0) == word(b, 0) && word(a, n - 8) == word(b, n - 8),
+        4..=7 => half(a, 0) == half(b, 0) && half(a, n - 4) == half(b, n - 4),
+        _ => {
+            let mut i = 0;
+            while i < n {
+                if a[i] != b[i] {
+                    return false;
+                }
+                i += 1;
+            }
+            true
         }
-        i += 1;
     }
-    true
 }
 
 /// The key for a newly populated slot `name`. The slots every raise
@@ -2559,7 +2567,81 @@ pub(crate) fn slot_key(name: &str) -> DictKey {
     DictKey(crate::stdlib::sys::intern_name(name))
 }
 
+/// The `BaseException` pseudo-slots after `args`, which raising and
+/// chaining populate: an exception instance's slots are laid out over them
+/// from its birth (see [`SlotStorage::exception`]).
+#[cfg(target_pointer_width = "64")]
+const EXC_TAIL: [&str; 4] = [
+    "__traceback__",
+    "__context__",
+    "__cause__",
+    "__suppress_context__",
+];
+
+/// The shared slot layout of an exception instance whose families' own
+/// fields are `extra`: `args`, `extra`, then [`EXC_TAIL`]. `None` for a
+/// shape without one (its slots are stored per key).
+#[cfg(target_pointer_width = "64")]
+fn exception_layout(extra: &[&'static str]) -> Option<SharedSlice<DictKey>> {
+    type Layout = std::sync::OnceLock<SharedSlice<DictKey>>;
+    static PLAIN: Layout = Layout::new();
+    static VALUE: Layout = Layout::new();
+    static MSG: Layout = Layout::new();
+    static NAME_OBJ: Layout = Layout::new();
+    static NAME: Layout = Layout::new();
+    let cell = match extra {
+        [] => &PLAIN,
+        ["value"] => &VALUE,
+        ["msg"] => &MSG,
+        ["name", "obj"] => &NAME_OBJ,
+        ["name"] => &NAME,
+        _ => return None,
+    };
+    Some(
+        cell.get_or_init(|| {
+            std::iter::once("args")
+                .chain(extra.iter().copied())
+                .chain(EXC_TAIL)
+                .map(slot_key)
+                .collect::<Vec<_>>()
+                .into()
+        })
+        .clone(),
+    )
+}
+
 impl SlotStorage {
+    /// The slots of a new exception instance: `args`, then its families'
+    /// own fields `extra` (`StopIteration.value`, `AttributeError`'s
+    /// `name` and `obj`), with the `BaseException` pseudo-slots laid out
+    /// unset after them, so a raise stores its traceback and chaining
+    /// links in place.
+    pub(crate) fn exception<const N: usize>(
+        args: Object,
+        extra: [(&'static str, Object); N],
+    ) -> Self {
+        let names: [&'static str; N] = std::array::from_fn(|i| extra[i].0);
+        #[cfg(target_pointer_width = "64")]
+        if let Some(layout) = exception_layout(&names) {
+            let mut values = Vec::with_capacity(layout.len());
+            values.push(args);
+            values.extend(extra.into_iter().map(|(_, v)| v));
+            let unset = layout.len() - values.len();
+            values.extend((0..unset).map(|_| Object::Unbound));
+            return Self {
+                data: SlotData::Fixed {
+                    layout,
+                    values: values.into_boxed_slice(),
+                },
+            };
+        }
+        let _ = names;
+        let mut entries = Vec::with_capacity(N + 2);
+        entries.push((slot_key("args"), args));
+        entries.extend(extra.into_iter().map(|(k, v)| (slot_key(k), v)));
+        Self::from_entries(entries)
+    }
+
     /// Storage holding exactly `entries` (distinct `str` keys, in slot
     /// order), built in one step.
     pub fn from_entries(entries: Vec<(DictKey, Object)>) -> Self {
@@ -3179,6 +3261,26 @@ impl SlotStorage {
         }
     }
 
+    /// Where slot `name` lives: its laid-out place (holding `Unbound`
+    /// while unset), else its populated slot. `None` when a store must
+    /// add the slot.
+    pub fn place_mut(&mut self, name: &str) -> Option<&mut Object> {
+        self.unpack();
+        let laid_out = match &self.data {
+            SlotData::Fixed { layout, .. } => Some(layout.iter().position(
+                |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)),
+            )?),
+            _ => None,
+        };
+        match laid_out {
+            Some(i) => match &mut self.data {
+                SlotData::Fixed { values, .. } => values.get_mut(i),
+                _ => None,
+            },
+            None => self.get_mut(name),
+        }
+    }
+
     pub fn insert(&mut self, name: &str, value: Object) -> Option<Object> {
         self.insert_with_key(name, value, || slot_key(name))
     }
@@ -3392,6 +3494,12 @@ impl SlotStorage {
         self.0.get(&crate::object::StrKey(name))
     }
 
+    /// The populated slot `name` (nothing is laid out here; see the
+    /// 64-bit variant).
+    pub fn place_mut(&mut self, name: &str) -> Option<&mut Object> {
+        self.get_mut(name)
+    }
+
     pub fn insert(&mut self, name: &str, value: Object) -> Option<Object> {
         self.insert_with_key(name, value, || slot_key(name))
     }
diff --git a/crates/weavepy-vm/src/vm_singletons.rs b/crates/weavepy-vm/src/vm_singletons.rs
index 53f6f668..8d7eafa7 100644
--- a/crates/weavepy-vm/src/vm_singletons.rs
+++ b/crates/weavepy-vm/src/vm_singletons.rs
@@ -598,7 +598,7 @@ pub fn thread_ident_is_live(id: u64) -> bool {
 #[derive(Clone, Debug)]
 pub struct ThreadHandles {
     pub frame_stack: crate::object::FrameStack,
-    pub exc_info_stack: Rc<RefCell<Vec<crate::error::PyException>>>,
+    pub exc_info_stack: Rc<RefCell<Vec<Object>>>,
     pub excepthook: Rc<RefCell<Object>>,
     pub unraisable_hook: Rc<RefCell<Object>>,
 }
diff --git a/crates/weavepy/src/lib.rs b/crates/weavepy/src/lib.rs
index d2835c35..9e627ca3 100644
--- a/crates/weavepy/src/lib.rs
+++ b/crates/weavepy/src/lib.rs
@@ -78,12 +78,13 @@ impl Error {
             Error::Runtime(vm::RuntimeError::PyException(exc)) => {
                 let mut s = String::new();
                 let _ = writeln!(s, "Traceback (most recent call last):");
-                // Tracebacks are accumulated from inner-most frame
-                // outward; print outer-most first to match CPython.
-                if exc.traceback.is_empty() {
+                // The `__traceback__` chain, outermost frame first, as
+                // CPython prints it.
+                let entries = exc.traceback_entries();
+                if entries.is_empty() {
                     let _ = writeln!(s, "  File \"{filename}\", line ?, in <module>");
                 } else {
-                    for entry in exc.traceback.iter().rev() {
+                    for entry in &entries {
                         let _ = writeln!(
                             s,
                             "  File \"{}\", line {}, in {}",
diff --git a/tests/regrtest/test_exception_slot_layout.py b/tests/regrtest/test_exception_slot_layout.py
new file mode 100644
index 00000000..9756fdb9
--- /dev/null
+++ b/tests/regrtest/test_exception_slot_layout.py
@@ -0,0 +1,276 @@
+"""Exception pseudo-slots laid out from birth, and handled-exception state.
+
+An exception instance's `args`, `__traceback__` and chaining links (and a
+family's own fields, such as `StopIteration.value`) live in a slot layout
+shared by every instance of the shape. Storing a field the layout doesn't
+name, deleting one, or reading one that was never set must behave as
+before. The handled-exception stack holds only the instances, so
+`sys.exc_info()`, bare `raise`, generator suspension inside handlers and
+`gen.throw()` chaining must still see the right exception.
+"""
+
+import copy
+import io
+import pickle
+import sys
+import unittest
+
+
+def raiser(exc):
+    raise exc
+
+
+def outer(exc):
+    raiser(exc)
+
+
+class AppError(Exception):
+    def __init__(self, code, msg):
+        super().__init__(msg)
+        self.code = code
+
+
+class SlottedError(Exception):
+    __slots__ = ("extra",)
+
+
+class SlotLayoutTest(unittest.TestCase):
+    def test_defaults_before_raise(self):
+        for e in (ValueError("x"), StopIteration(3), KeyError("k"), AppError(1, "m")):
+            self.assertIsNone(e.__traceback__)
+            self.assertIsNone(e.__context__)
+            self.assertIsNone(e.__cause__)
+            self.assertFalse(e.__suppress_context__)
+        self.assertEqual(StopIteration(3).value, 3)
+        self.assertIsNone(StopIteration().value)
+        self.assertEqual(StopIteration(1, 2).args, (1, 2))
+
+    def test_traceback_chain_across_frames(self):
+        try:
+            outer(ValueError("deep"))
+        except ValueError as e:
+            tb = e.__traceback__
+        names = []
+        while tb is not None:
+            names.append(tb.tb_frame.f_code.co_name)
+            tb = tb.tb_next
+        self.assertEqual(names, ["test_traceback_chain_across_frames", "outer", "raiser"])
+
+    def test_reraise_keeps_and_extends_traceback(self):
+        def middle():
+            try:
+                raiser(KeyError("k"))
+            except KeyError:
+                raise
+
+        try:
+            middle()
+        except KeyError as e:
+            tb = e.__traceback__
+        names = []
+        while tb is not None:
+            names.append(tb.tb_frame.f_code.co_name)
+            tb = tb.tb_next
+        self.assertEqual(names, ["test_reraise_keeps_and_extends_traceback", "middle", "raiser"])
+
+    def test_traceback_assignment(self):
+        try:
+            raiser(ValueError())
+        except ValueError as e:
+            err = e
+        err.__traceback__ = None
+        self.assertIsNone(err.__traceback__)
+        try:
+            raise err
+        except ValueError as e:
+            self.assertIsNotNone(e.__traceback__)
+            self.assertIsNone(e.__traceback__.tb_next)
+        with self.assertRaises(TypeError):
+            err.__traceback__ = 5
+        self.assertIs(err.with_traceback(None), err)
+        self.assertIsNone(err.__traceback__)
+
+    def test_chaining(self):
+        try:
+            try:
+                raise KeyError("inner")
+            except KeyError:
+                raise ValueError("outer")
+        except ValueError as e:
+            self.assertIsInstance(e.__context__, KeyError)
+            self.assertIsNone(e.__cause__)
+            self.assertFalse(e.__suppress_context__)
+        try:
+            try:
+                raise KeyError("inner")
+            except KeyError as k:
+                raise AppError(404, "missing") from None
+        except AppError as e:
+            self.assertIsNone(e.__cause__)
+            self.assertTrue(e.__suppress_context__)
+            self.assertIsInstance(e.__context__, KeyError)
+            self.assertEqual(e.code, 404)
+            self.assertEqual(e.args, ("missing",))
+        try:
+            raise ValueError("v") from TypeError("t")
+        except ValueError as e:
+            self.assertIsInstance(e.__cause__, TypeError)
+            self.assertTrue(e.__suppress_context__)
+        e = ValueError()
+        e.__context__ = KeyError()
+        e.__cause__ = TypeError()
+        e.__suppress_context__ = False
+        self.assertIsInstance(e.__context__, KeyError)
+        self.assertIsInstance(e.__cause__, TypeError)
+        self.assertFalse(e.__suppress_context__)
+        e.__context__ = None
+        self.assertIsNone(e.__context__)
+
+    def test_family_fields(self):
+        try:
+            object().missing
+        except AttributeError as e:
+            self.assertEqual(e.name, "missing")
+            self.assertIsInstance(e.obj, object)
+            self.assertIsNotNone(e.__traceback__)
+        try:
+            next(iter(()))
+        except StopIteration as e:
+            self.assertIsNone(e.value)
+            self.assertIsNotNone(e.__traceback__)
+
+        def gen():
+            yield 1
+            return "done"
+
+        g = gen()
+        next(g)
+        try:
+            next(g)
+        except StopIteration as e:
+            self.assertEqual(e.value, "done")
+            self.assertEqual(e.args, ("done",))
+        try:
+            raise OSError(2, "No such file", "f.txt")
+        except FileNotFoundError as e:
+            self.assertEqual((e.errno, e.strerror, e.filename), (2, "No such file", "f.txt"))
+            self.assertIsNotNone(e.__traceback__)
+        try:
+            raise SystemExit(3)
+        except SystemExit as e:
+            self.assertEqual(e.code, 3)
+            self.assertIsNotNone(e.__traceback__)
+        e = ImportError("no module", name="m", path="/p")
+        self.assertEqual((e.msg, e.name, e.path), ("no module", "m", "/p"))
+
+    def test_slotted_subclass(self):
+        try:
+            e = SlottedError("x")
+            e.extra = 5
+            raise e
+        except SlottedError as err:
+            self.assertEqual(err.extra, 5)
+            self.assertEqual(err.args, ("x",))
+            self.assertIsNotNone(err.__traceback__)
+        e = SlottedError()
+        with self.assertRaises(AttributeError):
+            e.extra
+
+    def test_copy_and_pickle(self):
+        try:
+            err = KeyError("seven")
+            err.code = 7
+            raise err
+        except KeyError as e:
+            err = e
+        for clone in (copy.copy(err), pickle.loads(pickle.dumps(err))):
+            self.assertEqual(clone.args, ("seven",))
+            self.assertEqual(clone.code, 7)
+            self.assertIsNone(clone.__traceback__)
+        si = pickle.loads(pickle.dumps(StopIteration(4)))
+        self.assertEqual((si.args, si.value), ((4,), 4))
+
+
+class HandledStateTest(unittest.TestCase):
+    def test_exc_info_nesting(self):
+        self.assertEqual(sys.exc_info(), (None, None, None))
+        try:
+            raise KeyError("a")
+        except KeyError as a:
+            self.assertIs(sys.exc_info()[1], a)
+            self.assertIs(sys.exception(), a)
+            try:
+                raise ValueError("b")
+            except ValueError as b:
+                self.assertIs(sys.exc_info()[1], b)
+                self.assertIs(b.__context__, a)
+                self.assertIs(sys.exc_info()[2], b.__traceback__)
+            self.assertIs(sys.exc_info()[1], a)
+        self.assertIsNone(sys.exception())
+
+    def test_bare_raise_from_helper(self):
+        def reraise():
+            raise
+
+        try:
+            try:
+                raise KeyError("k")
+            except KeyError as k:
+                original = k
+                reraise()
+        except KeyError as e:
+            self.assertIs(e, original)
+
+    def test_generator_suspended_in_handler(self):
+        def gen():
+            try:
+                raise KeyError("in gen")
+            except KeyError:
+                yield sys.exception()
+                yield sys.exception()
+
+        g = gen()
+        first = next(g)
+        self.assertIsInstance(first, KeyError)
+        self.assertIsNone(sys.exception())
+        try:
+            raise ValueError("outside")
+        except ValueError as outside:
+            self.assertIs(next(g), first)
+            self.assertIs(sys.exception(), outside)
+        g.close()
+
+    def test_throw_chains_suspended_handler(self):
+        def gen():
+            try:
+                raise KeyError("handled")
+            except KeyError:
+                yield 1
+
+        g = gen()
+        next(g)
+        with self.assertRaises(ValueError) as cm:
+            g.throw(ValueError("thrown"))
+        self.assertIsInstance(cm.exception.__context__, KeyError)
+
+    def test_unraisable_traceback_order(self):
+        def inner():
+            raise RuntimeError("from del")
+
+        class Noisy:
+            def __del__(self):
+                inner()
+
+        saved = sys.stderr
+        sys.stderr = buf = io.StringIO()
+        try:
+            Noisy()
+        finally:
+            sys.stderr = saved
+        text = buf.getvalue()
+        self.assertIn("RuntimeError: from del", text)
+        self.assertLess(text.index("__del__"), text.index("in inner"))
+
+
+if __name__ == "__main__":
+    unittest.main()

From 2b4c24f173b79cf574548544b656c85c98aacbf2 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 14:39:50 -0700
Subject: [PATCH 090/173] perf: trim generator creation, finish and exception
 slot stores

A compiled caller's `CALL` of a generator function now makes the
generator ahead of the other call shapes (`gen.send`, `next`, settled
builtins, pure leaves), none of which a plain function matches, and the
callee leaves the stack by a plain decrement when something else still
holds it. A generator that returns or is dropped marks itself finished
without a borrow of its state (the running state owns nothing; a drop
has the generator exclusively).

An exception's traceback head, chaining links and suppression flag are
stored in their laid-out places by one identity compare of the key
(`SlotStorage::exc_tail_mut`) instead of a name scan, and a new
exception's slots are written in place.

Per-iteration instructions vs the previous commit (macOS): fib(5)
coroutines 48256 -> 46731, a one-yield generator in a `for` 2985 ->
2850, `next(gen)` to exhaustion caught 7407 -> 7302, `raise
StopIteration` caught 3718 -> 3654, `raise ValueError(i)` caught 4088
-> 4050. Under callgrind against the branch base (both commits):
coroutines 48216 -> 46396, `raise ValueError(i)` 4234 -> 3852, `raise
StopIteration` 4190 -> 3449.
---
 crates/weavepy-vm/src/frame_jit.rs |  38 ++++---
 crates/weavepy-vm/src/gen_fast.rs  |   2 +-
 crates/weavepy-vm/src/lib.rs       |  50 +++++++---
 crates/weavepy-vm/src/object.rs    |  18 ++--
 crates/weavepy-vm/src/types.rs     | 154 ++++++++++++++++++++++-------
 5 files changed, 190 insertions(+), 72 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index f9bd8b00..447aafd5 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -4659,6 +4659,28 @@ unsafe extern "C" fn h_call(
             frame.pc = pc as u32;
             *(*st.sw).last = st.last;
         };
+        // A generator function's call makes its generator in place
+        // (`core_call` admits none), ahead of the shapes below, none of
+        // which a plain function matches.
+        if let Object::Function(f) = &*base.add(start) {
+            let c = &*f.code.as_ptr();
+            if (c.is_generator || c.is_coroutine || c.is_async_generator) && !st.sw.is_null() {
+                sync(st);
+                let sw = &mut *st.sw;
+                let interp = &mut *st.interp.cast_mut();
+                if !interp.core_gen_call(sw, pc) {
+                    if sw.pending.is_none() {
+                        sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+                    }
+                    return RELOAD;
+                }
+                return if finished_in_place(st, start) {
+                    0
+                } else {
+                    RELOAD
+                };
+            }
+        }
         // `gen.send(v)`: likewise, with the value sent in.
         if argc == 1
             && !st.sw.is_null()
@@ -4790,19 +4812,12 @@ unsafe extern "C" fn h_call(
             if st.sw.is_null() {
                 return CALL_DECLINED;
             }
-            // A generator function's call makes its generator in place
-            // (`core_call` admits none).
-            let gen_fn = matches!(&ops[0], Object::Function(f) if {
-                let c = &*f.code.as_ptr();
-                c.is_generator || c.is_coroutine || c.is_async_generator
-            });
+            // (A generator function's call was made above.)
             sync(st);
             let sw = &mut *st.sw;
             let saved_last = sw.last;
             let interp = &mut *st.interp.cast_mut();
-            let switched = if gen_fn {
-                interp.core_gen_call(sw, pc)
-            } else if python == 1 {
+            let switched = if python == 1 {
                 interp.core_call(sw, pc) || interp.core_gen_call(sw, pc)
             } else {
                 interp.core_new(sw, pc, st.snap_gen)
@@ -5502,9 +5517,10 @@ unsafe extern "C" fn h_raise(st: *mut State, pc: u64, len: u64, n: u64) -> u32 {
         let cause = (n == 2).then(|| st.stack.add(len - 1).read());
         let arg = st.stack.add(len - n).read();
         let interp = &mut *st.interp.cast_mut();
-        let globals = (*st.frame).globals.clone();
+        // (The frame holds its globals for the raise.)
+        let globals = &(*st.frame).globals;
         let raised = interp
-            .instantiate_raised_class(arg, &globals)
+            .instantiate_raised_class(arg, globals)
             .and_then(|arg| Interpreter::normalize_exception(arg, cause))
             .map(|mut exc| {
                 interp.attach_implicit_context(&mut exc);
diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index 6dfd8e8d..db1f9cbc 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -1077,7 +1077,7 @@ impl Interpreter {
             // generator finishes and its frame is released (anything that
             // dies with it queues its finalizer).
             GenStep::Returned(v) => {
-                *g.state.borrow_mut() = GeneratorState::Finished;
+                Self::finish_running_gen(g);
                 let frame: &mut Frame = &mut boxed;
                 self.recycle_frame_allocs(frame);
                 Self::release_finished_gen(g);
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 754d64fb..009745f8 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -8536,7 +8536,7 @@ impl Interpreter {
                 Ok(GenStep::Yielded(v))
             }
             Ok(FrameOutcome::Returned(v)) => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                Self::finish_running_gen(&gen);
                 let frame: &mut Frame = &mut boxed;
                 self.recycle_frame_allocs(frame);
                 Self::release_finished_gen(&gen);
@@ -21121,7 +21121,7 @@ impl Interpreter {
         let gen = done.gen.take().expect("a generator activation");
         let mut boxed = done.gen_box.take().expect("a generator activation");
         done.gen_frame = std::ptr::null_mut();
-        *gen.state.borrow_mut() = GeneratorState::Finished;
+        Self::finish_running_gen(&gen);
         self.recycle_frame_allocs(&mut boxed);
         Self::release_finished_gen(&gen);
         self.recycle_gen_box(boxed);
@@ -32228,7 +32228,7 @@ impl Interpreter {
             let new_tb = Object::Traceback(new_tb);
             inst.note_slot_store(&new_tb);
             let mut slots = inst.slots.borrow_mut();
-            match slots.place_mut("__traceback__") {
+            match slots.exc_tail_place(crate::types::ExcTail::Traceback) {
                 Some(slot) => {
                     let prev = std::mem::replace(slot, new_tb);
                     // The chain so far moves under the new head.
@@ -32400,14 +32400,15 @@ impl Interpreter {
     /// `False`), so only real values need storing.
     fn sync_exc_attrs(exc: &PyException) {
         if let Object::Instance(inst) = &exc.instance {
+            use crate::types::ExcTail;
             if let Some(cause) = exc.cause.as_ref() {
-                inst.slot_set("__cause__", cause.instance.clone());
+                drop(inst.exc_tail_set(ExcTail::Cause, cause.instance.clone()));
                 // Explicit cause suppresses __context__ rendering by
                 // default; user code can still set __suppress_context__.
-                inst.slot_set("__suppress_context__", Object::Bool(true));
+                drop(inst.exc_tail_set(ExcTail::SuppressContext, Object::Bool(true)));
             }
             if let Some(context) = exc.context.as_ref() {
-                inst.slot_set("__context__", context.instance.clone());
+                drop(inst.exc_tail_set(ExcTail::Context, context.instance.clone()));
             }
         }
     }
@@ -32497,12 +32498,17 @@ impl Interpreter {
             if matches!(c, Object::None) {
                 pe.cause = None;
                 if let Object::Instance(ref inst_rc) = pe.instance {
+                    use crate::types::ExcTail;
                     // An unset `__cause__` already reads None (a re-raised
                     // instance's earlier cause is cleared).
-                    if inst_rc.slots.borrow().get("__cause__").is_some() {
-                        inst_rc.slot_set("__cause__", Object::None);
-                    }
-                    inst_rc.slot_set("__suppress_context__", Object::Bool(true));
+                    let earlier = inst_rc
+                        .slots
+                        .borrow_mut()
+                        .exc_tail_place(ExcTail::Cause)
+                        .filter(|cause| !matches!(cause, Object::Unbound))
+                        .map(|cause| std::mem::replace(cause, Object::None));
+                    drop(earlier);
+                    drop(inst_rc.exc_tail_set(ExcTail::SuppressContext, Object::Bool(true)));
                 }
             } else {
                 let cpe = Self::normalize_exception(c, None)?;
@@ -43944,7 +43950,7 @@ impl Interpreter {
                 Ok(v)
             }
             Ok(FrameOutcome::Returned(v)) => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                Self::finish_running_gen(&gen);
                 let frame: &mut Frame = &mut boxed;
                 self.recycle_frame_allocs(frame);
                 Self::release_finished_gen(gen);
@@ -43991,6 +43997,21 @@ impl Interpreter {
         gc_trace::untrack_generator(gen);
     }
 
+    /// Mark `gen`, running (its frame out of its state), finished.
+    #[inline]
+    fn finish_running_gen(gen: &Rc<PyGenerator>) {
+        // SAFETY: the store runs no code (the displaced state is `Running`,
+        // which owns nothing) and no guard is live on the cell (`peek_mut`
+        // checks).
+        match unsafe { gen.state.peek_mut() } {
+            Some(state) if matches!(state, GeneratorState::Running) => unsafe {
+                std::ptr::write(state, GeneratorState::Finished);
+            },
+            Some(state) => *state = GeneratorState::Finished,
+            None => *gen.state.borrow_mut() = GeneratorState::Finished,
+        }
+    }
+
     fn park_suspended_boxed(gen: &Rc<PyGenerator>, boxed: Box<Frame>) {
         if let Some(py) = &boxed.py_frame {
             if py.gen_owner.borrow().is_none() {
@@ -57206,7 +57227,12 @@ impl Interpreter {
             take_args(&mut positional);
             self.start_generator_at(f, &code, positional, start)
         };
-        self.release(callee);
+        match callee {
+            // A function something else still holds (its module, its
+            // class) can't die here: a plain decrement.
+            Object::Function(f) if Rc::strong_count(&f) > 1 => drop(f),
+            callee => self.release(callee),
+        }
         frame.stack.push(gen);
         frame.pc = pc as u32 + 1;
         // SAFETY: the running activation's last-pc slot.
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index e33b02ec..97354ce9 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -5729,17 +5729,13 @@ impl Drop for PyGenerator {
         // pipeline (`chain(chain(chain(…)))`); dropping it inline recurses
         // one native stack frame per link and overflows on long chains, so
         // route it through the iterative trampoline below.
-        if let Ok(mut state) = self.state.try_borrow_mut() {
-            let prev = std::mem::replace(&mut *state, GeneratorState::Finished);
-            drop(state);
-            if matches!(
-                prev,
-                GeneratorState::Suspended(_)
-                    | GeneratorState::Created(_)
-                    | GeneratorState::Delegating(_)
-            ) {
-                defer_generator_state_drop(prev);
-            }
+        // (Exclusive: no borrow of the state can be live.)
+        let state = self.state.get_mut();
+        if matches!(
+            state,
+            GeneratorState::Suspended(_) | GeneratorState::Created(_) | GeneratorState::Delegating(_)
+        ) {
+            defer_generator_state_drop(std::mem::replace(state, GeneratorState::Finished));
         }
         crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64);
     }
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 347cffa8..b6632c4d 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -2525,23 +2525,49 @@ fn slot_name_eq(stored: &str, name: &str) -> bool {
     }
 }
 
+/// The slot names every raise populates (`args`, `__traceback__`, the
+/// chaining links) and the commonest family fields, in [`common_slot_keys`]
+/// order. `__traceback__` through `__suppress_context__` are [`ExcTail`]'s.
+const COMMON_SLOT_NAMES: [&str; 10] = [
+    "args",
+    "__traceback__",
+    "__context__",
+    "__cause__",
+    "__suppress_context__",
+    "message",
+    "value",
+    "msg",
+    "name",
+    "obj",
+];
+
+/// [`COMMON_SLOT_NAMES`], interned once.
+fn common_slot_keys() -> &'static [Object; 10] {
+    static KEYS: std::sync::OnceLock<[Object; 10]> = std::sync::OnceLock::new();
+    KEYS.get_or_init(|| COMMON_SLOT_NAMES.map(crate::stdlib::sys::intern_name))
+}
+
+/// A `BaseException` pseudo-slot, laid out at the end of every exception
+/// instance's slots (see [`SlotStorage::exception`]).
+#[derive(Clone, Copy)]
+pub(crate) enum ExcTail {
+    Traceback = 0,
+    Context = 1,
+    Cause = 2,
+    SuppressContext = 3,
+}
+
+impl ExcTail {
+    /// The slot's name.
+    pub(crate) fn name(self) -> &'static str {
+        COMMON_SLOT_NAMES[1 + self as usize]
+    }
+}
+
 /// The key for a newly populated slot `name`. The slots every raise
 /// populates (`args`, `__traceback__`, the chaining links) share one
 /// interned key each instead of allocating a string per exception.
 pub(crate) fn slot_key(name: &str) -> DictKey {
-    const COMMON: [&str; 10] = [
-        "args",
-        "__traceback__",
-        "__context__",
-        "__cause__",
-        "__suppress_context__",
-        "message",
-        "value",
-        "msg",
-        "name",
-        "obj",
-    ];
-    static KEYS: std::sync::OnceLock<[Object; 10]> = std::sync::OnceLock::new();
     // (By length first: one comparison settles each common name.)
     let common = match name.len() {
         3 if name == "msg" => Some(7),
@@ -2557,9 +2583,8 @@ pub(crate) fn slot_key(name: &str) -> DictKey {
         _ => None,
     };
     if let Some(i) = common {
-        debug_assert_eq!(COMMON[i], name);
-        let keys = KEYS.get_or_init(|| COMMON.map(crate::stdlib::sys::intern_name));
-        return DictKey(keys[i].clone());
+        debug_assert_eq!(COMMON_SLOT_NAMES[i], name);
+        return DictKey(common_slot_keys()[i].clone());
     }
     // Interned, as instance-dict keys are: a guard holding the interned
     // name settles a slot's key by identity, and no instance allocates
@@ -2567,20 +2592,14 @@ pub(crate) fn slot_key(name: &str) -> DictKey {
     DictKey(crate::stdlib::sys::intern_name(name))
 }
 
-/// The `BaseException` pseudo-slots after `args`, which raising and
-/// chaining populate: an exception instance's slots are laid out over them
-/// from its birth (see [`SlotStorage::exception`]).
+/// How many [`ExcTail`] pseudo-slots end an exception layout.
 #[cfg(target_pointer_width = "64")]
-const EXC_TAIL: [&str; 4] = [
-    "__traceback__",
-    "__context__",
-    "__cause__",
-    "__suppress_context__",
-];
+const EXC_TAIL_LEN: usize = 4;
 
 /// The shared slot layout of an exception instance whose families' own
-/// fields are `extra`: `args`, `extra`, then [`EXC_TAIL`]. `None` for a
-/// shape without one (its slots are stored per key).
+/// fields are `extra`: `args`, `extra`, then the [`ExcTail`] pseudo-slots,
+/// which raising and chaining populate. `None` for a shape without one
+/// (its slots are stored per key).
 #[cfg(target_pointer_width = "64")]
 fn exception_layout(extra: &[&'static str]) -> Option<SharedSlice<DictKey>> {
     type Layout = std::sync::OnceLock<SharedSlice<DictKey>>;
@@ -2601,7 +2620,7 @@ fn exception_layout(extra: &[&'static str]) -> Option<SharedSlice<DictKey>> {
         cell.get_or_init(|| {
             std::iter::once("args")
                 .chain(extra.iter().copied())
-                .chain(EXC_TAIL)
+                .chain(COMMON_SLOT_NAMES[1..=EXC_TAIL_LEN].iter().copied())
                 .map(slot_key)
                 .collect::<Vec<_>>()
                 .into()
@@ -2623,16 +2642,20 @@ impl SlotStorage {
         let names: [&'static str; N] = std::array::from_fn(|i| extra[i].0);
         #[cfg(target_pointer_width = "64")]
         if let Some(layout) = exception_layout(&names) {
-            let mut values = Vec::with_capacity(layout.len());
-            values.push(args);
-            values.extend(extra.into_iter().map(|(_, v)| v));
-            let unset = layout.len() - values.len();
-            values.extend((0..unset).map(|_| Object::Unbound));
+            // Written in place: `args`, the fields, then the unset tail.
+            let mut values = Box::<[Object]>::new_uninit_slice(layout.len());
+            values[0].write(args);
+            for (slot, (_, v)) in values[1..].iter_mut().zip(extra) {
+                slot.write(v);
+            }
+            for slot in &mut values[1 + N..] {
+                slot.write(Object::Unbound);
+            }
+            // SAFETY: every element was written above (the layout holds
+            // `args`, the `N` fields and the tail).
+            let values = unsafe { values.assume_init() };
             return Self {
-                data: SlotData::Fixed {
-                    layout,
-                    values: values.into_boxed_slice(),
-                },
+                data: SlotData::Fixed { layout, values },
             };
         }
         let _ = names;
@@ -3261,6 +3284,36 @@ impl SlotStorage {
         }
     }
 
+    /// The place of the exception pseudo-slot `which` (holding `Unbound`
+    /// while unset) in storage laid out over an exception layout (see
+    /// [`Self::exception`]): one identity compare of the laid-out key.
+    /// `None` for any other storage.
+    #[inline]
+    pub(crate) fn exc_tail_mut(&mut self, which: ExcTail) -> Option<&mut Object> {
+        let SlotData::Fixed { layout, values } = &mut self.data else {
+            return None;
+        };
+        let i = layout.len().checked_sub(EXC_TAIL_LEN)? + which as usize;
+        match (&layout[i].0, &common_slot_keys()[1 + which as usize]) {
+            (Object::Str(laid), Object::Str(key))
+                if crate::shared_value::SharedStr::ptr_eq(laid, key) =>
+            {
+                values.get_mut(i)
+            }
+            _ => None,
+        }
+    }
+
+    /// [`Self::place_mut`] of the exception pseudo-slot `which`, by
+    /// [`Self::exc_tail_mut`] when the storage has an exception layout.
+    #[inline]
+    pub(crate) fn exc_tail_place(&mut self, which: ExcTail) -> Option<&mut Object> {
+        if self.exc_tail_mut(which).is_some() {
+            return self.exc_tail_mut(which);
+        }
+        self.place_mut(which.name())
+    }
+
     /// Where slot `name` lives: its laid-out place (holding `Unbound`
     /// while unset), else its populated slot. `None` when a store must
     /// add the slot.
@@ -3500,6 +3553,18 @@ impl SlotStorage {
         self.get_mut(name)
     }
 
+    /// Never laid out here (see the 64-bit variant).
+    #[inline]
+    pub(crate) fn exc_tail_mut(&mut self, _which: ExcTail) -> Option<&mut Object> {
+        None
+    }
+
+    /// The populated pseudo-slot `which` (see the 64-bit variant).
+    #[inline]
+    pub(crate) fn exc_tail_place(&mut self, which: ExcTail) -> Option<&mut Object> {
+        self.get_mut(which.name())
+    }
+
     pub fn insert(&mut self, name: &str, value: Object) -> Option<Object> {
         self.insert_with_key(name, value, || slot_key(name))
     }
@@ -4019,6 +4084,21 @@ impl PyInstance {
         self.slots.borrow_mut().insert(name, value);
     }
 
+    /// [`Self::slot_set`] of an exception pseudo-slot: in its laid-out
+    /// place when the instance has one (see `SlotStorage::exc_tail_mut`).
+    /// Returns the displaced value (`Unbound` if unset), dropped by the
+    /// caller once the slots are released.
+    pub(crate) fn exc_tail_set(&self, which: ExcTail, value: Object) -> Object {
+        self.note_slot_store(&value);
+        let mut slots = self.slots.borrow_mut();
+        if let Some(place) = slots.exc_tail_mut(which) {
+            return std::mem::replace(place, value);
+        }
+        slots
+            .insert(which.name(), value)
+            .unwrap_or(Object::Unbound)
+    }
+
     /// Delete slot `name` from the side table; `false` when unset.
     pub fn slot_del(&self, name: &str) -> bool {
         self.slots.borrow_mut().remove(name).is_some()

From ce1fe92ff83d2eb171b9460d728e670c5a111d4d Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 15:16:44 -0700
Subject: [PATCH 091/173] perf: start an awaited coroutine as its call makes it

`await f(...)` of a coroutine function compiles to the call, then
`GET_AWAITABLE` (which passes a fresh coroutine through), the sent
`None` and the `SEND` that starts the coroutine. The call now starts it
at once through the same resume the `SEND` would run, with the caller
resting at the `SEND` where the coroutine's return lands, and a
compiled caller goes on at the await's `END_SEND` when the coroutine
returned.

Such a coroutine runs on the spine as soon as it exists, so its frame's
referents are roots and it can close no unreachable cycle until it
suspends: its handoff to the collector waits for its first suspension
(`PyGenerator::track_deferred`), and one that finishes first is never
tracked. `gc.is_tracked` answers as CPython's and tracks it from there.
The last release of a finished generator reads its state without a
borrow.

Per-iteration instructions (macOS): fib(5) coroutines 46731 -> 42589,
an unstarted generator made and dropped 1803 -> 1728. Suite
(instructions per call vs the previous commit): coroutines 0.942,
generators_tree 0.991, geomean 0.993.
---
 crates/weavepy-vm/src/frame_jit.rs      |  71 +++++++-
 crates/weavepy-vm/src/lib.rs            | 108 +++++++++--
 crates/weavepy-vm/src/object.rs         |   7 +
 crates/weavepy-vm/src/stdlib/gc_real.rs |  10 +-
 crates/weavepy-vm/src/vm_singletons.rs  |  20 +-
 tests/regrtest/test_await_call_entry.py | 231 ++++++++++++++++++++++++
 6 files changed, 422 insertions(+), 25 deletions(-)
 create mode 100644 tests/regrtest/test_await_call_entry.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 447aafd5..450b8e63 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -76,6 +76,10 @@ pub(crate) const RELOAD: u32 = 4;
 /// synced to the return with the value atop its stack: the caller
 /// finishes the return.
 const DIRECT_RET: u32 = 5;
+/// A `CALL` of a coroutine function followed by an `await` of its result
+/// started the coroutine, which returned (see [`gen_call_done`]): the
+/// await's `END_SEND` goes on, `[coroutine, result]` atop the stack.
+const AWAITED: u32 = 6;
 
 /// The heat per instruction of a code object at which it compiles (see
 /// `Slot::warm`; `WEAVEPY_FRAME_JIT_HOT` overrides it, a tuning aid).
@@ -3880,6 +3884,40 @@ unsafe fn run_switched_directly(st: &mut State) -> bool {
     }
 }
 
+/// The end of `h_call`'s generator-function call at `pc` (its operands
+/// from `start`), made by `Interpreter::core_gen_call`: the generator in
+/// place (`0`), or an `await` whose coroutine that call started (the
+/// switch in the state), run from here as [`h_send`] runs a resumed
+/// delegate: `AWAITED` when it returned into the await's `END_SEND`
+/// (`[coroutine, result]` from `start`), else `RELOAD`.
+///
+/// # Safety
+///
+/// `st` is a live state whose switch is non-null.
+unsafe fn gen_call_done(st: &mut State, pc: usize, start: usize) -> u32 {
+    // SAFETY: the caller's contract.
+    unsafe {
+        let sw = &*st.sw;
+        if !std::ptr::eq(sw.cur, st.frame) {
+            if sw.pending.is_none() && run_switched_directly(st) {
+                let frame = &*st.frame;
+                let send_pc = pc + 3;
+                let exit = send_pc + 1 + frame.code.instructions[send_pc].arg as usize;
+                if frame.pc as usize == exit && frame.stack.len() == start + 2 {
+                    st.len = start + 2;
+                    return AWAITED;
+                }
+            }
+            return RELOAD;
+        }
+        if finished_in_place(st, start) {
+            0
+        } else {
+            RELOAD
+        }
+    }
+}
+
 /// Whether the call whose operands started at `start` finished in place
 /// (a frameless `__init__`, a builtin's lane): nothing pending, the same
 /// activation running, its result alone in their place. Then the native
@@ -4674,11 +4712,7 @@ unsafe extern "C" fn h_call(
                     }
                     return RELOAD;
                 }
-                return if finished_in_place(st, start) {
-                    0
-                } else {
-                    RELOAD
-                };
+                return gen_call_done(st, pc, start);
             }
         }
         // `gen.send(v)`: likewise, with the value sent in.
@@ -4818,7 +4852,13 @@ unsafe extern "C" fn h_call(
             let saved_last = sw.last;
             let interp = &mut *st.interp.cast_mut();
             let switched = if python == 1 {
-                interp.core_call(sw, pc) || interp.core_gen_call(sw, pc)
+                if interp.core_call(sw, pc) {
+                    true
+                } else if interp.core_gen_call(sw, pc) {
+                    return gen_call_done(st, pc, start);
+                } else {
+                    false
+                }
             } else {
                 interp.core_new(sw, pc, st.snap_gen)
             };
@@ -9860,6 +9900,25 @@ impl<'a> Lower<'a> {
         self.b.ins().brif(r, other, &[], ran, &[]);
         // Raised or switched: the core loop takes it from the state.
         self.b.switch_to_block(other);
+        // `await f(...)` of a coroutine function: a coroutine the call
+        // started and that returned goes on at the await's `END_SEND`
+        // (see `gen_call_done`).
+        if let Some(send_pc) = crate::await_send_after(self.code, pc) {
+            let exit = send_pc + 1 + self.code.instructions[send_pc].arg as usize;
+            let start = self.depth - argc - 2;
+            if self.depths.get(exit).copied() == Some(start as i64 + 2) {
+                let awaited = self.b.create_block();
+                let rest = self.b.create_block();
+                let is = self.b.ins().icmp_imm(IntCC::Equal, r, i64::from(AWAITED));
+                self.b.ins().brif(is, awaited, &[], rest, &[]);
+                self.b.switch_to_block(awaited);
+                let depth = std::mem::replace(&mut self.depth, start + 2);
+                self.set_last(send_pc);
+                self.goto(exit);
+                self.depth = depth;
+                self.b.switch_to_block(rest);
+            }
+        }
         let declined = self
             .b
             .ins()
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 009745f8..5ff28080 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -13713,8 +13713,11 @@ impl Interpreter {
                                 } else if self.core_gen_call(sw, pc) {
                                     // The generator, made in place: no switch,
                                     // so the loop goes on (unless a release
-                                    // queued a finalizer).
-                                    if sw.pending.is_none() {
+                                    // queued a finalizer, or an `await`
+                                    // started the coroutine).
+                                    if sw.pending.is_none()
+                                        && std::ptr::eq(sw.cur.cast_const(), &raw const *frame)
+                                    {
                                         // SAFETY: the running activation (its
                                         // stack only shrank: no reallocation).
                                         len = unsafe { (*sw.cur).stack.len() };
@@ -44012,7 +44015,20 @@ impl Interpreter {
         }
     }
 
+    /// Hand a coroutine whose tracking was deferred (see
+    /// [`PyGenerator::track_deferred`]) to the collector.
+    #[cold]
+    #[inline(never)]
+    fn track_deferred_gen(gen: &Rc<PyGenerator>) {
+        gen.track_deferred.set(false);
+        gc_trace::track_generator(&Object::Coroutine(gen.clone()));
+    }
+
     fn park_suspended_boxed(gen: &Rc<PyGenerator>, boxed: Box<Frame>) {
+        // Suspended, it can close a cycle the collector must see.
+        if gen.track_deferred.get() {
+            Self::track_deferred_gen(gen);
+        }
         if let Some(py) = &boxed.py_frame {
             if py.gen_owner.borrow().is_none() {
                 *py.gen_owner.borrow_mut() = Some(Rc::downgrade(gen));
@@ -56864,6 +56880,16 @@ impl Interpreter {
     /// Wrap `frame`, `f`'s generator-family body just past
     /// `RETURN_GENERATOR`, in its generator object (tracked).
     fn wrap_started_generator(f: &PyFunction, code: &CodeObject, frame: Box<Frame>) -> Object {
+        let obj = Self::make_started_generator(f, code, frame);
+        // RFC 0024: generator frames can participate in reference cycles
+        // (a local that holds the generator itself), so track them like
+        // instances.
+        gc_trace::track_generator(&obj);
+        obj
+    }
+
+    /// [`Self::wrap_started_generator`] without tracking the generator.
+    fn make_started_generator(f: &PyFunction, code: &CodeObject, frame: Box<Frame>) -> Object {
         let kind = if code.is_coroutine {
             crate::object::CoroutineKind::Coroutine
         } else if code.is_async_generator {
@@ -56907,18 +56933,13 @@ impl Interpreter {
                 )
             }
         });
-        let obj = if code.is_coroutine {
+        if code.is_coroutine {
             Object::Coroutine(gen)
         } else if code.is_async_generator {
             Object::AsyncGenerator(gen)
         } else {
             Object::Generator(gen)
-        };
-        // RFC 0024: generator frames can participate in reference cycles
-        // (a local that holds the generator itself), so track them like
-        // instances.
-        gc_trace::track_generator(&obj);
-        obj
+        }
     }
 
     /// `iter(obj)` (`name` `__iter__`) or `obj.__await__()` for an
@@ -57196,6 +57217,12 @@ impl Interpreter {
             std::ptr::copy_nonoverlapping(args, v.as_mut_ptr().add(v.len()), nargs);
             v.set_len(v.len() + nargs);
         };
+        // `await f(...)`: the coroutine starts as it is made (see below).
+        let await_send = if code.is_coroutine && self.inline_calls_ok() {
+            await_send_after(&frame.code, pc)
+        } else {
+            None
+        };
         let gen = if let Some(cells) = cells {
             // The arguments move straight into the locals (no staging).
             let nlocals = code.varnames.len();
@@ -57217,7 +57244,18 @@ impl Interpreter {
             let mut gen_frame = Self::gen_frame_lean(f, code, locals, cells, start);
             gen_frame.stack = self.pooled_stack();
             let code = Rc::clone(&gen_frame.code);
-            Self::wrap_started_generator(f, &code, self.box_gen_frame(gen_frame))
+            let boxed = self.box_gen_frame(gen_frame);
+            if await_send.is_some() {
+                // Tracked if it suspends (see `PyGenerator::track_deferred`),
+                // or below if it doesn't start.
+                let obj = Self::make_started_generator(f, &code, boxed);
+                if let Object::Coroutine(g) = &obj {
+                    g.track_deferred.set(true);
+                }
+                obj
+            } else {
+                Self::wrap_started_generator(f, &code, boxed)
+            }
         } else {
             let mut positional = self.pooled_scratch();
             positional.extend(callee_receiver);
@@ -57233,13 +57271,45 @@ impl Interpreter {
             Object::Function(f) if Rc::strong_count(&f) > 1 => drop(f),
             callee => self.release(callee),
         }
+        let deferred = matches!(&gen, Object::Coroutine(g) if g.track_deferred.get());
         frame.stack.push(gen);
         frame.pc = pc as u32 + 1;
         // SAFETY: the running activation's last-pc slot.
         unsafe { *sw.last = pc };
+        // `await f(...)`: the coroutine starts at once, as the `SEND` after
+        // its `GET_AWAITABLE` (which passes a fresh coroutine through) would
+        // start it with the `None` sent. The caller rests at the `SEND`,
+        // where the coroutine's return lands; the caller of this function
+        // finds the switch in `sw`.
+        let caller: *mut Frame = frame;
         // SAFETY: the running thread's own flag (see `quiet_run`).
-        if unsafe { (*sw.maybe_dead).get() } {
+        let started = if unsafe { (*sw.maybe_dead).get() } {
             sw.pending = Some(CoreExit::Stop(LeafStop::Marked));
+            false
+        } else if let Some(send_pc) = await_send {
+            frame.stack.push(Object::None);
+            frame.pc = send_pc as u32;
+            // SAFETY: as above (the `LOAD_CONST` before the `SEND`).
+            unsafe { *sw.last = send_pc - 1 };
+            let started = self.core_gen_resume_one(sw, send_pc, InlineResume::Send);
+            if !started {
+                // SAFETY: still the running activation (nothing switched).
+                let frame = unsafe { &mut *caller };
+                frame.stack.pop();
+                frame.pc = pc as u32 + 1;
+                // SAFETY: as above.
+                unsafe { *sw.last = pc };
+            }
+            started
+        } else {
+            false
+        };
+        if deferred && !started {
+            // SAFETY: as above.
+            if let Some(obj @ Object::Coroutine(g)) = unsafe { (*caller).stack.last() } {
+                g.track_deferred.set(false);
+                gc_trace::track_generator(obj);
+            }
         }
         true
     }
@@ -66243,6 +66313,22 @@ fn yield_from_resend(instrs: &[weavepy_compiler::Instruction], pc: usize) -> Opt
     (instrs.get(target)?.op == OpCode::Send).then_some(target)
 }
 
+/// The `SEND` of an `await` of the `CALL` at `pc`'s result: the call is
+/// followed by `GET_AWAITABLE 0; LOAD_CONST None; SEND`.
+#[inline]
+pub(crate) fn await_send_after(code: &CodeObject, pc: usize) -> Option<usize> {
+    let [get, none, send] = code.instructions.get(pc + 1..pc + 4)? else {
+        return None;
+    };
+    (get.op == OpCode::GetAwaitable
+        && get.arg == 0
+        && none.op == OpCode::LoadConst
+        && matches!(code.constants.get(none.arg as usize), Some(Constant::None))
+        && send.op == OpCode::Send
+        && send.arg & GEN_SEND == 0)
+        .then_some(pc + 3)
+}
+
 /// Move the object at `src` to `dst` as one 16-byte copy: a move through
 /// a typed value (`pop` then `push`) splits it at the tag byte, and the
 /// 16-byte reload that follows stalls on store forwarding.
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 97354ce9..504fd96f 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -5534,6 +5534,12 @@ pub struct PyGenerator {
     /// [`crate::gc_trace::GcSlot`]); unregistered while it is still in a
     /// young set.
     pub gc_slot: crate::gc_trace::GcSlot,
+    /// Not yet handed to the collector: a coroutine an `await` started
+    /// as it was made runs on the spine (its frame's referents are
+    /// roots), so it can close no unreachable cycle until it first
+    /// suspends, which tracks it (see `Interpreter::park_suspended_boxed`).
+    /// One that finishes first is never tracked.
+    pub track_deferred: crate::sync::Cell<bool>,
 }
 
 /// The state of a [`PyGenerator`] that few carry.
@@ -5675,6 +5681,7 @@ impl PyGenerator {
             hooks_inited: crate::sync::Cell::new(false),
             finalize_ran: crate::sync::Cell::new(false),
             gc_slot: crate::gc_trace::GcSlot::new(),
+            track_deferred: crate::sync::Cell::new(false),
         }
     }
 
diff --git a/crates/weavepy-vm/src/stdlib/gc_real.rs b/crates/weavepy-vm/src/stdlib/gc_real.rs
index e2ddc654..3c8fe90d 100644
--- a/crates/weavepy-vm/src/stdlib/gc_real.rs
+++ b/crates/weavepy-vm/src/stdlib/gc_real.rs
@@ -506,8 +506,16 @@ fn object_is_tracked(target: &Object) -> bool {
             inst.ensure_gc_tracked();
             gc_trace::with_state(|s| s.is_tracked_instance(target))
         }
-        // A function, generator or coroutine may still be young.
+        // A function, generator or coroutine may still be young. A
+        // coroutine whose tracking is deferred (see
+        // `PyGenerator::track_deferred`) answers as CPython's and is
+        // tracked from here on.
         Object::Function(_) | Object::Generator(_) | Object::Coroutine(_) => {
+            if let Object::Generator(g) | Object::Coroutine(g) = target {
+                if g.track_deferred.replace(false) {
+                    gc_trace::track_generator(target);
+                }
+            }
             gc_trace::with_state(|s| s.is_tracked_instance(target))
         }
         // A list/dict/set born holding only scalars has its tracking
diff --git a/crates/weavepy-vm/src/vm_singletons.rs b/crates/weavepy-vm/src/vm_singletons.rs
index 8d7eafa7..fe8b3f1d 100644
--- a/crates/weavepy-vm/src/vm_singletons.rs
+++ b/crates/weavepy-vm/src/vm_singletons.rs
@@ -150,14 +150,20 @@ pub(crate) fn finalize_on_last_release<T: ?Sized + 'static>(arc: std::sync::Arc<
     } else if t == TypeId::of::<PyGenerator>() {
         // SAFETY: as above, for `PyGenerator`.
         let g = unsafe { Arc::from_raw(Arc::into_raw(arc).cast::<PyGenerator>()) };
+        let owes_for = |state: &GeneratorState| match state {
+            GeneratorState::Suspended(_) => true,
+            GeneratorState::Created(_) => g.kind == CoroutineKind::Coroutine,
+            GeneratorState::Finished | GeneratorState::Running | GeneratorState::Delegating(_) => {
+                false
+            }
+        };
+        // SAFETY: a read that runs no code (`peek_mut` rejects a live
+        // borrow or shared cells, which take the borrow).
         let owes = !g.finalize_ran.get()
-            && g.state.try_borrow().is_ok_and(|state| match &*state {
-                GeneratorState::Suspended(_) => true,
-                GeneratorState::Created(_) => g.kind == CoroutineKind::Coroutine,
-                GeneratorState::Finished
-                | GeneratorState::Running
-                | GeneratorState::Delegating(_) => false,
-            });
+            && match unsafe { g.state.peek_mut() } {
+                Some(state) => owes_for(state),
+                None => g.state.try_borrow().is_ok_and(|state| owes_for(&state)),
+            };
         // A plain generator suspended outside any handler of its own
         // (CPython's `gen_close` shortcut) just finishes, here: nothing
         // could observe the `GeneratorExit` its finalizer would throw.
diff --git a/tests/regrtest/test_await_call_entry.py b/tests/regrtest/test_await_call_entry.py
new file mode 100644
index 00000000..4d1fadd1
--- /dev/null
+++ b/tests/regrtest/test_await_call_entry.py
@@ -0,0 +1,231 @@
+"""`await f(...)` of a coroutine function starts the coroutine at once.
+
+The call that makes the coroutine also starts it, as the `SEND` after
+`GET_AWAITABLE` would. Everything an await shows must stay as before:
+results, exceptions and their tracebacks, PEP 479's conversion, the
+coroutine's frame and state while it runs, suspension through the await,
+recursion limits, and trace events.
+"""
+
+import sys
+import types
+import unittest
+
+
+def drive(coro):
+    try:
+        while True:
+            coro.send(None)
+    except StopIteration as e:
+        return e.value
+
+
+@types.coroutine
+def suspend(value=None):
+    return (yield value)
+
+
+async def fib(n):
+    if n <= 1:
+        return n
+    return await fib(n - 1) + await fib(n - 2)
+
+
+async def fail(exc):
+    raise exc
+
+
+async def await_fail(exc):
+    return await fail(exc)
+
+
+class Box:
+    def __init__(self, v):
+        self.v = v
+
+    async def get(self):
+        return self.v
+
+    async def total(self):
+        return await self.get() + await self.get()
+
+
+class AwaitCallEntryTest(unittest.TestCase):
+    def test_results(self):
+        self.assertEqual(drive(fib(15)), 610)
+        self.assertEqual(drive(Box(21).total()), 42)
+
+    def test_exception_and_traceback(self):
+        try:
+            drive(await_fail(KeyError("k")))
+        except KeyError as e:
+            tb = e.__traceback__
+        names = []
+        while tb is not None:
+            names.append(tb.tb_frame.f_code.co_name)
+            tb = tb.tb_next
+        self.assertEqual(names[-3:], ["drive", "await_fail", "fail"])
+
+    def test_stop_iteration_becomes_runtime_error(self):
+        with self.assertRaises(RuntimeError) as cm:
+            drive(await_fail(StopIteration(1)))
+        self.assertIn("coroutine raised StopIteration", str(cm.exception))
+        self.assertIsInstance(cm.exception.__cause__, StopIteration)
+
+    def test_caught_in_caller(self):
+        async def caller():
+            try:
+                await fail(ValueError("v"))
+            except ValueError as e:
+                return e.args
+        self.assertEqual(drive(caller()), ("v",))
+
+    def test_frame_and_state_while_running(self):
+        seen = {}
+
+        async def inner():
+            frame = sys._getframe()
+            coro = frame.f_generator
+            seen["running"] = coro.cr_running
+            seen["frame"] = coro.cr_frame is frame
+            seen["name"] = coro.__qualname__
+            seen["back"] = frame.f_back.f_code.co_name
+            seen["line"] = frame.f_back.f_lineno
+            return 7
+
+        async def outer():
+            return await inner()
+
+        self.assertEqual(drive(outer()), 7)
+        self.assertTrue(seen["running"])
+        self.assertTrue(seen["frame"])
+        self.assertTrue(seen["name"].endswith("inner"))
+        self.assertEqual(seen["back"], "outer")
+        self.assertEqual(seen["line"], outer.__code__.co_firstlineno + 1)
+
+    def test_suspension_through_the_await(self):
+        async def inner():
+            got = await suspend("first")
+            got2 = await suspend("second")
+            return got + got2
+
+        async def outer():
+            return await inner() * 2
+
+        coro = outer()
+        self.assertEqual(coro.send(None), "first")
+        self.assertEqual(coro.cr_await.cr_code.co_name, "inner")
+        self.assertFalse(coro.cr_await.cr_running)
+        self.assertEqual(coro.send(3), "second")
+        with self.assertRaises(StopIteration) as cm:
+            coro.send(4)
+        self.assertEqual(cm.exception.value, 14)
+
+    def test_throw_and_close_while_suspended(self):
+        log = []
+
+        async def inner():
+            try:
+                await suspend()
+            except KeyError:
+                log.append("caught")
+                return "recovered"
+            finally:
+                log.append("finally")
+
+        async def outer():
+            return await inner()
+
+        coro = outer()
+        coro.send(None)
+        with self.assertRaises(StopIteration) as cm:
+            coro.throw(KeyError())
+        self.assertEqual(cm.exception.value, "recovered")
+        self.assertEqual(log, ["caught", "finally"])
+        coro = outer()
+        coro.send(None)
+        coro.close()
+        self.assertEqual(log, ["caught", "finally", "finally"])
+
+    def test_gc_tracking(self):
+        import gc
+        import weakref
+
+        class Node:
+            pass
+
+        seen = {}
+
+        async def inner(node):
+            coro = sys._getframe().f_generator
+            seen["tracked"] = gc.is_tracked(coro)
+            # A cycle through the coroutine, live while it is suspended.
+            node.coro = coro
+            await suspend()
+
+        async def outer(node):
+            await inner(node)
+
+        node = Node()
+        coro = outer(node)
+        coro.send(None)
+        self.assertTrue(seen["tracked"])
+        self.assertTrue(gc.is_tracked(node.coro))
+        ref = weakref.ref(node)
+        del node, coro
+        gc.collect()
+        self.assertIsNone(ref())
+
+    def test_recursion_limit(self):
+        async def deep(n):
+            return await deep(n + 1)
+
+        with self.assertRaises(RecursionError):
+            drive(deep(0))
+
+    def test_renamed_function(self):
+        async def named():
+            return sys._getframe().f_generator.__name__
+
+        named.__name__ = "renamed"
+
+        async def outer():
+            return await named()
+
+        self.assertEqual(drive(outer()), "renamed")
+
+    def test_trace_events(self):
+        events = []
+
+        async def inner():
+            return 1
+
+        async def outer():
+            return await inner()
+
+        def tracer(frame, event, arg):
+            if frame.f_code in (inner.__code__, outer.__code__):
+                events.append((frame.f_code.co_name, event))
+            return tracer
+
+        sys.settrace(tracer)
+        try:
+            drive(outer())
+        finally:
+            sys.settrace(None)
+        self.assertEqual(
+            events,
+            [
+                ("outer", "call"),
+                ("outer", "line"),
+                ("inner", "call"),
+                ("inner", "line"),
+                ("inner", "return"),
+                ("outer", "exception"),
+                ("outer", "return"),
+            ],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

From 383023c6979a13e60c3f96f2ce21d6640985cf5f Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 15:42:21 -0700
Subject: [PATCH 092/173] perf: lay out the slots of exceptions with a Python
 __init__

An exception class with its own `__init__` is constructed through the
lean instance path, which seeded `args` into a bare instance's per-key
slots; the raise then grew them for the traceback and chaining links.
The seeded instance is now laid out as a built-in construction's
(`SlotStorage::exception`), so `AppError(404, "missing")` raised `from
None` fills its slots in place.

Seeding `args` (and `BaseException.__init__` storing them) also handed
every such instance to the collector, since a tuple is never atomic.
Only a non-atomic argument can close a cycle through the instance, as
the built-in construction already judges, so the tracking now waits
for one.

Suite (instructions per call vs the previous commit): exceptions 0.982,
context_managers 0.998.
---
 crates/weavepy-vm/src/builtin_types.rs       | 18 +++++++--
 crates/weavepy-vm/src/lib.rs                 | 15 +++++++-
 tests/regrtest/test_exception_slot_layout.py | 40 ++++++++++++++++++++
 3 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs
index 685e3afe..04f197ef 100644
--- a/crates/weavepy-vm/src/builtin_types.rs
+++ b/crates/weavepy-vm/src/builtin_types.rs
@@ -4483,7 +4483,16 @@ pub(crate) fn exc_init(args: &[Object]) -> Result<Object, RuntimeError> {
             if is_subclass_by_name(&inst_rc.cls(), "StopIteration") {
                 inst_rc.slot_set("value", rest.first().cloned().unwrap_or(Object::None));
             }
-            inst_rc.slot_set("args", Object::new_tuple(rest));
+            // Only a non-atomic argument can close a cycle through the
+            // instance (as at construction, see `build_exception_instance`).
+            if rest.iter().any(|x| !crate::gc_trace::is_atomic(x)) {
+                inst_rc.ensure_gc_tracked();
+            }
+            let old = inst_rc
+                .slots
+                .borrow_mut()
+                .insert("args", Object::new_tuple(rest));
+            drop(old);
         }
         Ok(Object::None)
     }
@@ -4869,9 +4878,10 @@ pub fn make_exception_with_class(class: Rc<TypeObject>, message: impl Into<Strin
 /// A built-in exception of `class` whose `args` is `(arg,)`, or `()`
 /// for `None`: the shared tail of the Rust-side constructors.
 ///
-/// The pseudo-slots land in one vector sized for the traceback and
-/// chaining links a raise adds next, and the class's families come
-/// from its memoised flags rather than an MRO walk. CPython keeps no
+/// The pseudo-slots are laid out with room for the traceback and
+/// chaining links a raise adds next (see `SlotStorage::exception`), and
+/// the class's families come from its memoised flags rather than an MRO
+/// walk. CPython keeps no
 /// `message` attribute: `str()` and [`exception_message`] derive the
 /// text from `args` on demand, so nothing is formatted here.
 pub(crate) fn exception_from_parts(class: Rc<TypeObject>, arg: Option<Object>) -> Object {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 5ff28080..3b99097f 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -54895,7 +54895,20 @@ impl Interpreter {
     /// class's `__init__` never calls `super().__init__`.
     fn seed_exception_args(inst: &Object, args: impl Iterator<Item = Object>) {
         if let Object::Instance(i) = inst {
-            i.slot_set("args", Object::new_tuple(args.collect()));
+            let args = Object::new_tuple(args.collect());
+            if i.slots.borrow().is_empty_default() {
+                // Laid out as a built-in construction's (see
+                // `SlotStorage::exception`), so a raise fills its slots in
+                // place. As there, only a non-atomic argument can close a
+                // cycle through the instance.
+                if matches!(&args, Object::Tuple(t) if t.iter().any(|x| !gc_trace::is_atomic(x)))
+                {
+                    i.ensure_gc_tracked();
+                }
+                *i.slots.borrow_mut() = crate::types::SlotStorage::exception(args, []);
+            } else {
+                i.slot_set("args", args);
+            }
         }
     }
 
diff --git a/tests/regrtest/test_exception_slot_layout.py b/tests/regrtest/test_exception_slot_layout.py
index 9756fdb9..80b92b39 100644
--- a/tests/regrtest/test_exception_slot_layout.py
+++ b/tests/regrtest/test_exception_slot_layout.py
@@ -163,6 +163,46 @@ def gen():
         e = ImportError("no module", name="m", path="/p")
         self.assertEqual((e.msg, e.name, e.path), ("no module", "m", "/p"))
 
+    def test_python_init(self):
+        class NoSuper(Exception):
+            def __init__(self, a, b):
+                self.total = a + b
+
+        e = NoSuper(1, 2)
+        self.assertEqual(e.args, (1, 2))
+        self.assertEqual(e.total, 3)
+        e = AppError(404, "missing")
+        self.assertEqual(e.args, ("missing",))
+        try:
+            try:
+                {}["k"]
+            except KeyError:
+                raise AppError(5, "five") from None
+        except AppError as err:
+            self.assertEqual((err.code, err.args), (5, ("five",)))
+            self.assertTrue(err.__suppress_context__)
+            self.assertIsInstance(err.__context__, KeyError)
+            self.assertIsNotNone(err.__traceback__)
+
+    def test_cycle_through_args(self):
+        import gc
+        import weakref
+
+        class Holder:
+            pass
+
+        class Err(Exception):
+            def __init__(self, holder):
+                super().__init__(holder)
+
+        h = Holder()
+        h.err = Err(h)
+        ref = weakref.ref(h)
+        del h
+        gc.collect()
+        self.assertIsNone(ref())
+        self.assertTrue(gc.is_tracked(Err(Holder())))
+
     def test_slotted_subclass(self):
         try:
             e = SlottedError("x")

From 75982fb038a9dced9919474b6f56f4a9ec48e79d Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 16:18:38 -0700
Subject: [PATCH 093/173] perf: raise a cooling attribute miss without the
 resolution passes

A `LOAD_ATTR` site whose generic load failed cools down, and each later
execution ran the full lookup again before raising (`try: o.x except
AttributeError:`, EAFP probes). While such a site cools down, a plain
instance with no attribute of that name anywhere
(`attr_certainly_missing`, which `getattr` with a default already uses)
gets the `AttributeError` the default lookup ends with, `name` and `obj`
set, directly.

The site's cooldown count records why it cools down: it steps down two
at a time from an odd count after a failed load and from an even one
after an unspecializable success or a deopt, so sites whose loads
succeed don't pay for the probe.

Per-iteration instructions (macOS): `o.missing` caught 10532 -> 8439.
Suite (instructions per call vs the previous commit): exceptions 0.940;
deepcopy 0.999, geomean 0.998 (vs two commits back, exceptions 0.923).
---
 crates/weavepy-vm/src/lib.rs               |  28 ++++--
 tests/regrtest/test_attribute_miss_site.py | 100 +++++++++++++++++++++
 2 files changed, 123 insertions(+), 5 deletions(-)
 create mode 100644 tests/regrtest/test_attribute_miss_site.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 3b99097f..2486ab88 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -46979,10 +46979,16 @@ impl Interpreter {
                 // descriptor from one); that side effect must not run ahead
                 // of the lookup it's meant to accelerate.
                 specialize::record_specialize_attempt(op_idx);
+                // A site cools down two steps at a time from an odd count
+                // when its load failed, an even one otherwise (see the
+                // `Cooldown` arm).
                 let decision = if result.is_ok() {
-                    specialize::attempt_specialize_load_attr(&obj, &name)
+                    match specialize::attempt_specialize_load_attr(&obj, &name) {
+                        IC::Cooldown(n) => IC::Cooldown(n.saturating_mul(2) & !1),
+                        decision => decision,
+                    }
                 } else {
-                    IC::Cooldown(COOLDOWN)
+                    IC::Cooldown(2 * COOLDOWN + 1)
                 };
                 frame.code.caches.set(cache_pc, decision);
                 if matches!(decision, IC::Cooldown(_)) {
@@ -46993,12 +46999,23 @@ impl Interpreter {
                 result
             }
             IC::Cooldown(n) => {
-                let next = if n > 0 {
-                    IC::Cooldown(n - 1)
+                let next = if n >= 2 {
+                    IC::Cooldown(n - 2)
                 } else {
                     IC::Empty
                 };
                 frame.code.caches.set(cache_pc, next);
+                // A site cooling down after a failed load (an odd count; `try:
+                // o.x except AttributeError:`): an attribute certainly missing
+                // raises the `AttributeError` the default lookup ends with,
+                // without its resolution passes.
+                if let (1, Some(name)) = (n & 1, frame.code.names.get(name_idx as usize)) {
+                    if attr_certainly_missing(&receiver, name) {
+                        let err = crate::error::attribute_error_named(&receiver, name);
+                        self.release(receiver);
+                        return Err(err);
+                    }
+                }
                 let name = self.name_at(&frame.code, name_idx)?;
                 self.load_attr(&receiver, &name)
             }
@@ -47021,10 +47038,11 @@ impl Interpreter {
         receiver: Object,
     ) -> Result<Object, RuntimeError> {
         specialize::record_miss(OpCode::LoadAttr as u8);
+        // (An even count: see `specialized_load_attr`'s `Cooldown` arm.)
         frame
             .code
             .caches
-            .set(cache_pc, weavepy_compiler::InlineCache::Cooldown(COOLDOWN));
+            .set(cache_pc, weavepy_compiler::InlineCache::Cooldown(2 * COOLDOWN));
         let name = self.name_at(&frame.code, name_idx)?;
         self.load_attr(&receiver, &name)
     }
diff --git a/tests/regrtest/test_attribute_miss_site.py b/tests/regrtest/test_attribute_miss_site.py
new file mode 100644
index 00000000..e762a512
--- /dev/null
+++ b/tests/regrtest/test_attribute_miss_site.py
@@ -0,0 +1,100 @@
+"""A `LOAD_ATTR` site that keeps failing raises its `AttributeError` early.
+
+A site whose generic load failed cools down; while it does, a plain
+instance with no such attribute anywhere gets the `AttributeError` the
+default lookup ends with, without the resolution passes. Anything that
+makes the attribute exist, or changes how it is looked up, must be seen
+at once.
+"""
+
+import unittest
+
+
+class Plain:
+    pass
+
+
+def read(o):
+    return o.missing
+
+
+def probe(o, n):
+    hits = 0
+    for _ in range(n):
+        try:
+            read(o)
+        except AttributeError as e:
+            assert e.name == "missing" and e.obj is o
+        else:
+            hits += 1
+    return hits
+
+
+class AttributeMissSiteTest(unittest.TestCase):
+    def test_error_fields(self):
+        o = Plain()
+        for _ in range(300):
+            with self.assertRaises(AttributeError) as cm:
+                read(o)
+        self.assertEqual(cm.exception.name, "missing")
+        self.assertIs(cm.exception.obj, o)
+        self.assertEqual(str(cm.exception), "'Plain' object has no attribute 'missing'")
+
+    def test_instance_attribute_appears(self):
+        o = Plain()
+        self.assertEqual(probe(o, 300), 0)
+        o.missing = 5
+        self.assertEqual(probe(o, 10), 10)
+        del o.missing
+        self.assertEqual(probe(o, 10), 0)
+
+    def test_class_attribute_appears(self):
+        class C:
+            pass
+
+        o = C()
+        self.assertEqual(probe(o, 300), 0)
+        C.missing = 7
+        self.assertEqual(probe(o, 10), 10)
+        del C.missing
+        self.assertEqual(probe(o, 10), 0)
+
+    def test_getattr_hook_appears(self):
+        class C:
+            pass
+
+        o = C()
+        self.assertEqual(probe(o, 300), 0)
+        C.__getattr__ = lambda self, name: 1
+        self.assertEqual(probe(o, 10), 10)
+        del C.__getattr__
+        self.assertEqual(probe(o, 10), 0)
+
+    def test_property_raising(self):
+        calls = []
+
+        class C:
+            @property
+            def missing(self):
+                calls.append(1)
+                raise AttributeError("from property")
+
+        o = C()
+        self.assertEqual(probe(Plain(), 300), 0)
+        for _ in range(10):
+            with self.assertRaises(AttributeError) as cm:
+                read(o)
+            self.assertEqual(str(cm.exception), "from property")
+        self.assertEqual(len(calls), 10)
+
+    def test_receivers_change(self):
+        class Has:
+            missing = "here"
+
+        self.assertEqual(probe(Plain(), 300), 0)
+        self.assertEqual(probe(Has(), 10), 10)
+        self.assertEqual(probe(Plain(), 10), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 649d7f6e8016079379b0f6419bb9ac18d133d87b Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 16:42:30 -0700
Subject: [PATCH 094/173] perf: enter an awaited coroutine without re-checking
 its fresh state

The call that starts an awaited coroutine went through the `SEND`'s
resume, which located the coroutine under a pushed `None`, re-read its
state and re-ran the checks a suspended generator needs. A coroutine
the call just made passes all of them but its code's, so it now enters
through the commit half of that resume (`inline_gen_commit`, which
`try_inline_gen` shares) after that one check and the recursion limit.

Per-iteration instructions (macOS): fib(5) coroutines 42589 -> 40307.
Suite (instructions per call vs the previous commit): coroutines 0.975,
async_tree 0.997, geomean 1.000.
---
 crates/weavepy-vm/src/lib.rs | 100 +++++++++++++++++++++++++++++------
 1 file changed, 84 insertions(+), 16 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 2486ab88..35bf75a2 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -8407,11 +8407,41 @@ impl Interpreter {
         // Committed.
         let prev_state = std::mem::replace(state, GeneratorState::Running);
         let g = g.clone();
-        let (GeneratorState::Suspended(mut boxed) | GeneratorState::Created(mut boxed)) =
-            prev_state
-        else {
+        let (GeneratorState::Suspended(boxed) | GeneratorState::Created(boxed)) = prev_state else {
             unreachable!("checked above");
         };
+        let sent = if send {
+            frame.stack.pop().unwrap_or(Object::None)
+        } else {
+            Object::None
+        };
+        Some(self.inline_gen_commit(
+            frame,
+            shell,
+            pc,
+            mode,
+            arg,
+            (g, boxed, first_resume, sent),
+            guard,
+        ))
+    }
+
+    /// [`Self::try_inline_gen`]'s commit, once the generator `g` (its
+    /// frame `boxed`, running from here) passed its checks: the sent value
+    /// goes in (unless a fast step already consumed one), the consumer's
+    /// operands leave by `mode`, and the activation is built.
+    #[allow(clippy::too_many_arguments)]
+    #[inline(always)]
+    fn inline_gen_commit(
+        &mut self,
+        frame: &mut Frame,
+        shell: &mut QuietShell<'_>,
+        pc: usize,
+        mode: InlineResume,
+        arg: u32,
+        (g, mut boxed, first_resume, sent): (Rc<PyGenerator>, Box<Frame>, bool, Object),
+        guard: crate::recursion::Guard,
+    ) -> Box<InlineAct> {
         let gf: &mut Frame = &mut boxed;
         // A native activation parked at the yield is rebuilt into the
         // interpreted suspension resumed here.
@@ -8419,11 +8449,6 @@ impl Interpreter {
         crate::tier2::materialize_parked(gf);
         gf.gen_first_resume = first_resume;
         // (A fast step that stopped partway already consumed it.)
-        let sent = if send {
-            frame.stack.pop().unwrap_or(Object::None)
-        } else {
-            Object::None
-        };
         if !std::mem::take(&mut gf.sent_consumed) {
             gf.push(sent);
         }
@@ -8472,7 +8497,7 @@ impl Interpreter {
         act.call_pc = pc;
         act.caller_pending = self.lean_pending_enter(frame, shell, pc);
         act.exc_depth = self.exc_info_len();
-        Some(act)
+        act
     }
 
     /// Finish an inline generator resume that left the quiet loop with
@@ -57318,16 +57343,10 @@ impl Interpreter {
             sw.pending = Some(CoreExit::Stop(LeafStop::Marked));
             false
         } else if let Some(send_pc) = await_send {
-            frame.stack.push(Object::None);
-            frame.pc = send_pc as u32;
             // SAFETY: as above (the `LOAD_CONST` before the `SEND`).
             unsafe { *sw.last = send_pc - 1 };
-            let started = self.core_gen_resume_one(sw, send_pc, InlineResume::Send);
+            let started = self.core_await_enter(sw, send_pc);
             if !started {
-                // SAFETY: still the running activation (nothing switched).
-                let frame = unsafe { &mut *caller };
-                frame.stack.pop();
-                frame.pc = pc as u32 + 1;
                 // SAFETY: as above.
                 unsafe { *sw.last = pc };
             }
@@ -57345,6 +57364,55 @@ impl Interpreter {
         true
     }
 
+    /// The `await` (its `SEND` at `send_pc`) of the coroutine atop `sw`'s
+    /// running activation, which the activation's call just made: started
+    /// at once as the `SEND` would start it (`try_inline_gen`'s `Send`
+    /// resume, whose checks a fresh coroutine passes but for its code's),
+    /// the caller resting past the `SEND`. `false` touches nothing.
+    fn core_await_enter(&mut self, sw: &mut CoreSwitch, send_pc: usize) -> bool {
+        let mut tmp = None;
+        // SAFETY: see `CoreSwitch` (as in `core_gen_resume_one`).
+        let (frame, shell) = unsafe {
+            let depth = (*sw.inl).len();
+            let (frame, _, shell) = sw.activation(depth, &mut tmp);
+            (&mut *frame, &mut *shell.cast::<QuietShell<'_>>())
+        };
+        let Some(Object::Coroutine(g)) = frame.stack.last() else {
+            return false;
+        };
+        let g = g.clone();
+        // SAFETY: just made; nothing else reaches its state yet.
+        let state = unsafe { &mut *g.state.as_ptr() };
+        if !matches!(state, GeneratorState::Created(boxed) if Self::lean_gen_code_ok(&boxed.code)) {
+            return false;
+        }
+        // Past the recursion limit the `SEND` raises.
+        let crate::recursion::Enter::Ok(guard) = crate::recursion::enter_with(sw.depth_cell) else {
+            return false;
+        };
+        let GeneratorState::Created(boxed) = std::mem::replace(state, GeneratorState::Running)
+        else {
+            unreachable!("checked above");
+        };
+        let jump = frame.code.instructions[send_pc].arg;
+        let act = self.inline_gen_commit(
+            frame,
+            shell,
+            send_pc,
+            InlineResume::Send,
+            jump | GEN_SEND,
+            (g, boxed, true, Object::None),
+            guard,
+        );
+        let gen_frame = act.gen_frame;
+        // SAFETY: as above.
+        unsafe { push_fast(&mut *sw.inl, act) };
+        sw.cur = gen_frame;
+        sw.scratch = usize::MAX;
+        sw.last = &raw mut sw.scratch;
+        true
+    }
+
     /// Bootstrap a generator/coroutine *code object* frame that was not
     /// entered through a `PyFunction` call — `eval(co)` / `exec(co)` of
     /// PyCF_ALLOW_TOP_LEVEL_AWAIT module code (RFC 0052). Runs the frame

From 24c28a5b3879dec5b60584778416b77486a8c547 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 08:43:32 -0700
Subject: [PATCH 095/173] perf: trim the direct call's checks and its return's
 releases

A compiled caller's direct call (frame_jit::direct_call) paid about 570
instructions in its helper. Several of its steps did more than they
needed:

- The helper read the CALL's argument count back out of the code; the
  native code now passes it with the pc.
- A site hit compared the code, the self slot and the body separately;
  each cached callee now has one key (the code's address with the self
  slot in its low bit), zero while it has no body.
- The lean-call gate (Interpreter::inline_calls_ok) is tested once per
  burst (CoreSwitch::calls_ok). Its inputs are fixed for a burst except
  the eval-frame recorder, whose changes now bump the loop generation
  that every direct call compares.
- The tier-2 test reads the code's compiled flag first.
- The frame JIT's stats flag is one load instead of a OnceLock read.
- A finished activation's locals (release_locals, shared with
  core_return) are released by tag and pointer with the refcount bias
  tested once per run, not copied out and dropped one by one.
- The clean return skips the callee state's drop glue (a return raises
  nothing) and releases the callable with one shared-owner decrement.

A direct method call probe drops from 569 to 517 instructions in the
helper; go retires 2.9% fewer instructions, the suite 0.6%.

(Instruction counts measured on e58c3c3, before the rebase onto
649d7f6.)
---
 crates/weavepy-vm/src/frame_jit.rs | 83 ++++++++++++++++++++++++------
 crates/weavepy-vm/src/lib.rs       | 45 +++++++++++++++-
 crates/weavepy-vm/src/trace.rs     |  4 ++
 3 files changed, 114 insertions(+), 18 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 450b8e63..6f3f0fc4 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -783,12 +783,25 @@ mod stats {
 
     type Counts = HashMap<(String, String), [u64; 2]>;
 
-    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    /// `0` until decided, then `1` off or `2` on (one load per test: every
+    /// direct call's return tests it).
+    static ON: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
     static COUNTS: Mutex<Option<Counts>> = Mutex::new(None);
 
     #[inline(always)]
     pub(super) fn enabled() -> bool {
-        *ON.get_or_init(|| {
+        match ON.load(std::sync::atomic::Ordering::Relaxed) {
+            1 => false,
+            2 => true,
+            _ => decide(),
+        }
+    }
+
+    #[cold]
+    #[inline(never)]
+    fn decide() -> bool {
+        static ONCE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+        let on = *ONCE.get_or_init(|| {
             let on = std::env::var_os("WEAVEPY_FRAME_JIT_STATS").is_some();
             if on {
                 extern "C" fn dump() {
@@ -815,7 +828,9 @@ mod stats {
                 unsafe { libc::atexit(dump) };
             }
             on
-        })
+        });
+        ON.store(if on { 2 } else { 1 }, std::sync::atomic::Ordering::Relaxed);
+        on
     }
 
     /// [`super::run_pushed_directly`] declined, for `why`: `None`.
@@ -3963,6 +3978,17 @@ pub(crate) struct DirectSite {
     alt_native: usize,
     alt_missing: u32,
     alt_has_self: bool,
+    /// The hit keys: each callee's code address with the self slot in its
+    /// low bit, `0` while it has no body (one compare per lookup).
+    key: usize,
+    alt_key: usize,
+}
+
+/// A [`DirectSite`] key: `code`'s address, with `has_self` in its low bit
+/// (code objects are word-aligned).
+#[inline(always)]
+fn site_key(code: usize, has_self: bool) -> usize {
+    code | usize::from(has_self)
 }
 
 const DS_NATIVE: i32 = std::mem::offset_of!(DirectSite, native) as i32;
@@ -3994,6 +4020,11 @@ impl DirectSite {
         self.missing = missing as u32;
         self.native = native;
         self.retry = if native == 0 { 64 } else { 0 };
+        self.key = if native == 0 {
+            0
+        } else {
+            site_key(self.code, has_self)
+        };
     }
 
     /// Record a call of `code` with a native body as the site's first
@@ -4010,6 +4041,8 @@ impl DirectSite {
             self.alt_native = self.native;
             self.alt_missing = self.missing;
             self.alt_has_self = self.has_self;
+            self.alt_key = self.key;
+            self.key = 0;
         }
         self.fill(code, has_self, native, missing);
     }
@@ -4278,16 +4311,20 @@ unsafe extern "C" fn h_call_direct(
     st: *mut State,
     code: *const CodeObject,
     ext: *const CodeConstObjects,
-    pc: u64,
+    pc_argc: u64,
     len: u64,
     site: *mut DirectSite,
 ) -> u32 {
-    // SAFETY: as `h_call`'s; `site` is this `CALL`'s own cache.
+    // SAFETY: as `h_call`'s; `site` is this `CALL`'s own cache, and the
+    // `CALL`'s argument count (the high half of `pc_argc`) fits the stack
+    // (the native code checked its depth).
     unsafe {
         let s = &mut *st;
-        let argc = (&(*code).instructions)[pc as usize].arg as usize;
+        let pc = pc_argc & u64::from(u32::MAX);
+        let argc = (pc_argc >> 32) as usize;
         let len_u = len as usize;
-        if len_u >= argc + 2 && !s.sw.is_null() {
+        debug_assert!(len_u >= argc + 2);
+        if !s.sw.is_null() {
             let start = len_u - argc - 2;
             if let Some(r) = direct_call(s, pc as usize, start, len_u, &mut *site, None) {
                 return r;
@@ -4343,10 +4380,10 @@ unsafe fn direct_call(
         // `def` binds the same way, and code without free variables takes
         // no closure.)
         let at = Rc::as_ptr(code_rc) as usize;
-        let (native, packed) = if site.code == at && site.has_self == has_self && site.native != 0
-        {
+        let key = site_key(at, has_self);
+        let (native, packed) = if site.key == key {
             (site.native, site.missing as usize)
-        } else if site.alt_code == at && site.alt_has_self == has_self && site.alt_native != 0 {
+        } else if site.alt_key == key {
             (site.alt_native, site.alt_missing as usize)
         } else {
             let nargs = len - start - 2 + usize::from(has_self);
@@ -4382,13 +4419,14 @@ unsafe fn direct_call(
         let code: &CodeObject = code_rc;
         // (Native code runs only in a burst, which free-threaded mode
         // never starts.)
+        // (Tier 2's compiled code is the commonest test's rare answer.)
         if nested >= DIRECT_CALL_DEPTH
             || crate::hot_gates::loop_gen() != st.snap_gen
             || !native.enters_at(0)
-            || !interp.inline_calls_ok()
-            || !(crate::tier2::jit_off_for_process()
-                || code.jit_hint.is_not_jitable()
-                || !code.jit_hint.is_compiled())
+            || !(*st.sw).calls_ok
+            || (code.jit_hint.is_compiled()
+                && !code.jit_hint.is_not_jitable()
+                && !crate::tier2::jit_off_for_process())
         {
             return None;
         }
@@ -4592,8 +4630,12 @@ unsafe fn direct_run<const PUSHED: bool>(
                 }
                 st.stack.add(start).write(v);
                 caller.stack.set_len(start + 1);
+                // A function something else still holds (its class, its
+                // module) can't die here: a plain decrement.
                 match callable {
-                    Object::Function(f) if Rc::strong_count(&f) > 1 => drop(f),
+                    Object::Function(f) if crate::rc::try_release_shared(Rc::as_ptr(&f)) => {
+                        std::mem::forget(f);
+                    }
                     callable => interp.release(callable),
                 }
                 sw.cur = st.frame;
@@ -4611,6 +4653,10 @@ unsafe fn direct_run<const PUSHED: bool>(
                     sw.scratch = usize::MAX;
                 }
                 *sw.last = pc;
+                // (A return raised nothing: the callee's state holds no
+                // error to drop.)
+                debug_assert!(cst.err.is_none());
+                std::mem::forget(cst);
                 if sw.pending.is_some() {
                     return RELOAD;
                 }
@@ -9871,10 +9917,15 @@ impl<'a> Lower<'a> {
         self.b.ins().store(FLAGS, left, site_at, DS_RETRY);
         self.b.ins().jump(general, &[]);
         self.b.switch_to_block(direct);
+        // (The direct helper takes the argument count with the pc.)
+        let pc_argc = self
+            .b
+            .ins()
+            .iconst(types::I64, (pc as i64) | ((argc as i64) << 32));
         let r = self
             .call(
                 h_call_direct as *const () as usize,
-                &[self.st, code, ext, pcv, len, site_at],
+                &[self.st, code, ext, pc_argc, len, site_at],
                 true,
             )
             .expect("returns");
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 35bf75a2..59d813ee 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -7728,7 +7728,8 @@ impl Interpreter {
                 let stop = match pending.take() {
                     Some(stop) => stop,
                     None => {
-                        let mut sw = CoreSwitch::enter(roots, frame, shell, last);
+                        let calls_ok = self.inline_calls_ok();
+                        let mut sw = CoreSwitch::enter(roots, frame, shell, last, calls_ok);
                         let stop = self.leaf_burst(&mut sw, snap_gen);
                         if sw.switched() {
                             return FrameEv::Switched(stop);
@@ -65991,6 +65992,11 @@ struct CoreSwitch {
     /// The observer generation the burst started under (see
     /// [`Interpreter::core_pending_exit`]).
     obs_gen: u64,
+    /// `Interpreter::inline_calls_ok` when the burst started: its inputs
+    /// hold for a burst (a native stack's greenlet, the process's flags)
+    /// or bump the loop generation when they change (the eval-frame
+    /// recorder), which every direct call compares first.
+    calls_ok: bool,
 }
 
 impl CoreSwitch {
@@ -66000,6 +66006,7 @@ impl CoreSwitch {
         frame: &mut Frame,
         shell: &mut QuietShell<'_>,
         last: &mut usize,
+        calls_ok: bool,
     ) -> Self {
         let frame: *mut Frame = frame;
         let shell: *mut QuietShell<'static> = std::ptr::from_mut(shell).cast();
@@ -66024,6 +66031,7 @@ impl CoreSwitch {
             maybe_dead: roots.maybe_dead,
             depth_cell: crate::recursion::depth_cell(),
             obs_gen: crate::trace::observer_gen(),
+            calls_ok,
         }
     }
 
@@ -66485,8 +66493,41 @@ unsafe fn release_locals(locals: &mut Vec<Object>) {
     // SAFETY: the first `n` values are initialized.
     unsafe {
         locals.set_len(0);
+        release_run(base, n);
+    }
+}
+
+/// Release the `n` values from `base` on (each read out once), as
+/// [`drop_hot`] releases one: the bias is tested once for the run, and a
+/// shared owner's decrement reads only the tag and the pointer (a moved
+/// out copy of each value showed in every return's profile).
+///
+/// # Safety
+///
+/// The `n` values are initialized, owned by the caller, and never read
+/// again.
+#[inline(always)]
+unsafe fn release_run(base: *const Object, n: usize) {
+    // SAFETY: the caller's contract.
+    unsafe {
+        if !crate::rc::refcounts_biased() {
+            for k in 0..n {
+                drop(base.add(k).read());
+            }
+            return;
+        }
         for k in 0..n {
-            drop_hot(base.add(k).read());
+            let at = base.add(k);
+            if let Some(word) = (*at).strong_word() {
+                // While the bias holds this thread alone updates counts.
+                let w = &*word;
+                let c = w.load(std::sync::atomic::Ordering::Relaxed);
+                if c != 1 {
+                    w.store(c - 1, std::sync::atomic::Ordering::Relaxed);
+                } else {
+                    drop(at.read());
+                }
+            }
         }
     }
 }
diff --git a/crates/weavepy-vm/src/trace.rs b/crates/weavepy-vm/src/trace.rs
index 17d15409..d901f7ba 100644
--- a/crates/weavepy-vm/src/trace.rs
+++ b/crates/weavepy-vm/src/trace.rs
@@ -482,11 +482,15 @@ static EVAL_FRAME_RECORD: Mutex<Option<Object>> = Mutex::new(None);
 pub fn set_eval_frame_record(list: Object) {
     *EVAL_FRAME_RECORD.lock().unwrap() = Some(list);
     EVAL_FRAME_RECORD_ACTIVE.store(true, Ordering::Release);
+    // A running burst's direct calls test the gate once per burst (see
+    // `CoreSwitch::calls_ok`): the new generation ends it.
+    crate::hot_gates::bump_loop_gen();
 }
 
 pub fn set_eval_frame_default() {
     EVAL_FRAME_RECORD_ACTIVE.store(false, Ordering::Release);
     *EVAL_FRAME_RECORD.lock().unwrap() = None;
+    crate::hot_gates::bump_loop_gen();
 }
 
 #[inline]

From 54e99406432f36e5a4ff50c134a004d9cdae7ff7 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 09:58:48 -0700
Subject: [PATCH 096/173] perf: let a direct call's site cache a frameless leaf
 callee too

A compiled caller's CALL of a frameless leaf (a pure or effect leaf with
no native body) went through h_call: the generator, next() and builtin
checks, the callee's classification, then core_pure_call's checks,
including its site's call-shape slot. A polymorphic site that also had a
compiled callee paid direct_body's eligibility checks on every leaf call
first.

The direct call's site (DirectSite) now also caches one leaf callee, by
the same code-and-self-slot key, when its parameters are positional
only: the native code takes the direct helper whenever the site has a
body or a leaf, and a leaf hit evaluates the callee in place through
Interpreter::site_leaf_eval, which keeps only the checks that can change
between calls (the leaf verdict, its warm-up, the defaults, the
recursion limit, droppable operands). A leaf that declines takes the
ordinary activation path (h_call's, now call_activation), and every
64th decline checks whether its code has compiled since, which then
takes the direct lane as before.

mini_interpreter retires 2.7% fewer instructions and raytrace 0.6%;
the rest of the suite moves within layout noise.

(Instruction counts measured on e58c3c3, before the rebase onto
649d7f6.)
---
 crates/weavepy-vm/src/frame_jit.rs | 219 ++++++++++++++++++++++-------
 crates/weavepy-vm/src/lib.rs       |  53 +++++++
 2 files changed, 225 insertions(+), 47 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 6f3f0fc4..b7c01fe8 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3982,6 +3982,15 @@ pub(crate) struct DirectSite {
     /// low bit, `0` while it has no body (one compare per lookup).
     key: usize,
     alt_key: usize,
+    /// A frameless leaf callee the site evaluates in place (see
+    /// [`leaf_site_call`]): its code (a counted reference, `0` none), key,
+    /// and defaults taken.
+    leaf_code: usize,
+    leaf_key: usize,
+    leaf_missing: u32,
+    /// Nonzero while the site has a body or a leaf to take: the native
+    /// code tries the direct helper first then.
+    hot: usize,
 }
 
 /// A [`DirectSite`] key: `code`'s address, with `has_self` in its low bit
@@ -3991,12 +4000,12 @@ fn site_key(code: usize, has_self: bool) -> usize {
     code | usize::from(has_self)
 }
 
-const DS_NATIVE: i32 = std::mem::offset_of!(DirectSite, native) as i32;
+const DS_HOT: i32 = std::mem::offset_of!(DirectSite, hot) as i32;
 const DS_RETRY: i32 = std::mem::offset_of!(DirectSite, retry) as i32;
 
 impl Drop for DirectSite {
     fn drop(&mut self) {
-        for code in [self.code, self.alt_code] {
+        for code in [self.code, self.alt_code, self.leaf_code] {
             if code != 0 {
                 // SAFETY: each holds the count a fill took.
                 unsafe { drop(Rc::from_raw(code as *const CodeObject)) };
@@ -4025,6 +4034,23 @@ impl DirectSite {
         } else {
             site_key(self.code, has_self)
         };
+        self.hot = self.native | self.leaf_key;
+    }
+
+    /// Record a frameless leaf callee running `code` (with or without its
+    /// self slot), whose last `missing` parameters take defaults.
+    fn fill_leaf(&mut self, code: &Rc<CodeObject>, has_self: bool, missing: usize) {
+        let at = Rc::as_ptr(code) as usize;
+        if at != self.leaf_code {
+            let old = std::mem::replace(&mut self.leaf_code, Rc::into_raw(code.clone()) as usize);
+            if old != 0 {
+                // SAFETY: `old` held the count an earlier fill took.
+                unsafe { drop(Rc::from_raw(old as *const CodeObject)) };
+            }
+        }
+        self.leaf_key = site_key(at, has_self);
+        self.leaf_missing = missing as u32;
+        self.hot = self.native | self.leaf_key;
     }
 
     /// Record a call of `code` with a native body as the site's first
@@ -4326,7 +4352,7 @@ unsafe extern "C" fn h_call_direct(
         debug_assert!(len_u >= argc + 2);
         if !s.sw.is_null() {
             let start = len_u - argc - 2;
-            if let Some(r) = direct_call(s, pc as usize, start, len_u, &mut *site, None) {
+            if let Some(r) = direct_call::<true>(s, pc as usize, start, len_u, &mut *site, None) {
                 return r;
             }
         }
@@ -4334,6 +4360,48 @@ unsafe extern "C" fn h_call_direct(
     }
 }
 
+/// Whether a site may evaluate a call of `f` (running `code`) with `nargs`
+/// positional arguments as a frameless leaf ([`leaf_site_call`]): a leaf
+/// with only positional parameters, at most eight, `nargs` of them given.
+/// How many take defaults.
+fn site_leaf_fits(f: &crate::object::PyFunction, code: &CodeObject, nargs: usize) -> Option<usize> {
+    let missing = (code.arg_count as usize).checked_sub(nargs)?;
+    (crate::fn_is_leaf(f)
+        && !Interpreter::has_extended_params(code)
+        && code.arg_count <= 8
+        && Interpreter::leaf_code_ok(code))
+    .then_some(missing)
+}
+
+/// A `CALL`'s (its operands from `start` up a `len`-deep stack) callee,
+/// the frameless leaf its site cached (see [`DirectSite::fill_leaf`]),
+/// whose last `missing` parameters take defaults, evaluated in place as
+/// `h_call` evaluates a leaf callee (`Interpreter::site_leaf_eval`):
+/// `true` with the result at `start` (`st.len` past it), `false` declined
+/// untouched.
+///
+/// # Safety
+///
+/// As [`direct_call`]'s; the callee is a plain function whose code is the
+/// site's leaf's.
+#[inline(never)]
+unsafe fn leaf_site_call(st: &mut State, start: usize, len: usize, missing: usize) -> bool {
+    // SAFETY: the caller's contract.
+    unsafe {
+        let base = st.stack;
+        let ops = std::slice::from_raw_parts(base.add(start), len - start);
+        let Some(r) = (*st.interp).site_leaf_eval(ops, missing, st.depth_cell) else {
+            return false;
+        };
+        for k in start..len {
+            crate::drop_hot(base.add(k).read());
+        }
+        base.add(start).write(r);
+        st.len = start + 1;
+        true
+    }
+}
+
 /// [`direct_call`] out of line, for the colder lanes (a property getter,
 /// an instance operator's method), which share one copy of it.
 #[inline(never)]
@@ -4346,7 +4414,7 @@ unsafe fn direct_call_out(
     binop: Option<(BinOpKind, bool, &Rc<crate::types::TypeObject>)>,
 ) -> Option<u32> {
     // SAFETY: the caller's contract.
-    unsafe { direct_call(st, pc, start, len, site, binop) }
+    unsafe { direct_call::<false>(st, pc, start, len, site, binop) }
 }
 
 /// The `CALL` at `pc` (its operands from `start` up a `len`-deep stack) of
@@ -4360,7 +4428,7 @@ unsafe fn direct_call_out(
 /// (`binop`) that leaves the direct return to the core loop carries the
 /// operator, whose `TypeError` its `NotImplemented` becomes there.
 #[inline(always)]
-unsafe fn direct_call(
+unsafe fn direct_call<const CALL: bool>(
     st: &mut State,
     pc: usize,
     start: usize,
@@ -4387,6 +4455,21 @@ unsafe fn direct_call(
             (site.alt_native, site.alt_missing as usize)
         } else {
             let nargs = len - start - 2 + usize::from(has_self);
+            let leaf = CALL && site.leaf_key == key;
+            if leaf {
+                // The site's frameless leaf, evaluated in place.
+                if leaf_site_call(st, start, len, site.leaf_missing as usize) {
+                    return Some(0);
+                }
+                // Declined (a leaf that keeps bailing, or one not warm
+                // yet): an ordinary call, unless a body compiled since,
+                // which every few declines check for.
+                if site.retry != 0 {
+                    site.retry -= 1;
+                    return Some(call_activation_out(st, pc, start, len));
+                }
+                site.retry = 64;
+            }
             let (native, missing) = direct_body(f, code_rc, nargs).map_or((0, 0), |(n, m)| {
                 (std::ptr::from_ref::<Native>(n) as usize, m)
             });
@@ -4396,6 +4479,18 @@ unsafe fn direct_call(
                 if site.native == 0 {
                     site.fill(code_rc, has_self, 0, 0);
                 }
+                if leaf {
+                    return Some(call_activation_out(st, pc, start, len));
+                }
+                if CALL {
+                    if let Some(missing) = site_leaf_fits(f, code_rc, nargs) {
+                        site.fill_leaf(code_rc, has_self, missing);
+                        if leaf_site_call(st, start, len, missing) {
+                            return Some(0);
+                        }
+                        return Some(call_activation_out(st, pc, start, len));
+                    }
+                }
                 return None;
             }
             site.promote(code_rc, has_self, native, missing);
@@ -4886,46 +4981,7 @@ unsafe extern "C" fn h_call(
             }
         }
         if python != 0 {
-            // A Python callee's activation, switched to as the arm does
-            // (the frame synced first; a declined switch leaves the call
-            // to the quiet loop).
-            if st.sw.is_null() {
-                return CALL_DECLINED;
-            }
-            // (A generator function's call was made above.)
-            sync(st);
-            let sw = &mut *st.sw;
-            let saved_last = sw.last;
-            let interp = &mut *st.interp.cast_mut();
-            let switched = if python == 1 {
-                if interp.core_call(sw, pc) {
-                    true
-                } else if interp.core_gen_call(sw, pc) {
-                    return gen_call_done(st, pc, start);
-                } else {
-                    false
-                }
-            } else {
-                interp.core_new(sw, pc, st.snap_gen)
-            };
-            if !switched && sw.pending.is_none() {
-                sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
-            }
-            // A compiled callee runs from here, and its return continues
-            // this code natively (anything else leaves for the core loop).
-            if switched && sw.pending.is_none() && !std::ptr::eq(sw.cur, st.frame) {
-                if let Some(r) = run_pushed_directly(st, pc, start, saved_last) {
-                    return r;
-                }
-                if run_callee_directly(st, start) {
-                    st.len = start + 1;
-                    return 0;
-                }
-            }
-            if switched && finished_in_place(st, start) {
-                return 0;
-            }
-            return RELOAD;
+            return call_activation(st, pc, start, len, python);
         }
         // A builtin type's call (`range(a, b)`, `float(x)`, `list(xs)`), as
         // the core loop's arms run it: in place when nothing it does can
@@ -5066,6 +5122,75 @@ unsafe extern "C" fn h_call(
     }
 }
 
+/// [`call_activation`] of a plain function out of line, for a direct
+/// call's site whose frameless leaf declined (`h_call` keeps its own copy
+/// in line).
+///
+/// # Safety
+///
+/// As `h_call`'s.
+#[inline(never)]
+unsafe fn call_activation_out(st: &mut State, pc: usize, start: usize, len: usize) -> u32 {
+    // SAFETY: the caller's contract.
+    unsafe { call_activation(st, pc, start, len, 1) }
+}
+
+/// The `CALL` at `pc` (its operands from `start` up a `len`-deep stack) of
+/// a Python callee (`python` 1, a function or a bound method over one) or
+/// a Python class's constructor (`python` 2), its activation switched to
+/// as the core loop's arm switches it (the frame synced first; a declined
+/// switch leaves the call to the quiet loop): `h_call`'s results.
+///
+/// # Safety
+///
+/// As `h_call`'s.
+#[inline(always)]
+unsafe fn call_activation(st: &mut State, pc: usize, start: usize, len: usize, python: u8) -> u32 {
+    // SAFETY: the caller's contract.
+    unsafe {
+        if st.sw.is_null() {
+            return CALL_DECLINED;
+        }
+        let frame = &mut *st.frame;
+        frame.stack.set_len(len);
+        frame.pc = pc as u32;
+        *(*st.sw).last = st.last;
+        let sw = &mut *st.sw;
+        let saved_last = sw.last;
+        let interp = &mut *st.interp.cast_mut();
+        // (A generator function's call was made by `h_call` before.)
+        let switched = if python == 1 {
+            if interp.core_call(sw, pc) {
+                true
+            } else if interp.core_gen_call(sw, pc) {
+                return gen_call_done(st, pc, start);
+            } else {
+                false
+            }
+        } else {
+            interp.core_new(sw, pc, st.snap_gen)
+        };
+        if !switched && sw.pending.is_none() {
+            sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+        }
+        // A compiled callee runs from here, and its return continues
+        // this code natively (anything else leaves for the core loop).
+        if switched && sw.pending.is_none() && !std::ptr::eq(sw.cur, st.frame) {
+            if let Some(r) = run_pushed_directly(st, pc, start, saved_last) {
+                return r;
+            }
+            if run_callee_directly(st, start) {
+                st.len = start + 1;
+                return 0;
+            }
+        }
+        if switched && finished_in_place(st, start) {
+            return 0;
+        }
+        RELOAD
+    }
+}
+
 /// `len(x)` of an exact container or string.
 fn len_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
     // SAFETY: reads between two instructions (the lengths run no code).
@@ -9907,8 +10032,8 @@ impl<'a> Lower<'a> {
         self.b.append_block_param(called, types::I32);
         self.b.ins().brif(is_fn, fresh, &[], general, &[]);
         self.b.switch_to_block(fresh);
-        let native = self.b.ins().load(types::I64, FLAGS, site_at, DS_NATIVE);
-        self.b.ins().brif(native, direct, &[], waiting, &[]);
+        let hot = self.b.ins().load(types::I64, FLAGS, site_at, DS_HOT);
+        self.b.ins().brif(hot, direct, &[], waiting, &[]);
         self.b.switch_to_block(waiting);
         let retry = self.b.ins().load(types::I32, FLAGS, site_at, DS_RETRY);
         self.b.ins().brif(retry, counted, &[], direct, &[]);
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 59d813ee..d9ee9a44 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -19850,6 +19850,59 @@ impl Interpreter {
         self.pure_leaf_call(code_rc, f, &args[..arity], effect, depth)
     }
 
+    /// [`Self::core_pure_call`] for a `CALL` site of the frame JIT that
+    /// cached its leaf callee (see `frame_jit::leaf_site_call`): the site
+    /// settled the callee's shape (positional parameters only, at most
+    /// eight, the last `missing` of them from the defaults), so only the
+    /// checks that can change from call to call remain.
+    pub(crate) fn site_leaf_eval(
+        &self,
+        ops: &[Object],
+        missing: usize,
+        depth_cell: *const std::cell::Cell<usize>,
+    ) -> Option<Object> {
+        let Object::Function(f) = ops.first()? else {
+            return None;
+        };
+        // SAFETY: GIL-serialized raw read of the function's code cell (see
+        // `PyFunction::code`), while the caller's stack keeps the function
+        // alive.
+        let code_rc: &Rc<CodeObject> = unsafe { &*f.code.as_ptr() };
+        let pure = code_is_pure_leaf(code_rc);
+        let effect = !pure && code_is_effect_leaf(code_rc);
+        if !(pure || effect) || !pure_leaf_warm(code_rc) {
+            return None;
+        }
+        if missing > 0
+            && (f.defaults.len() < missing
+                || (f.defaults_maybe_overridden() && f.slot("__defaults__").is_some()))
+        {
+            return None;
+        }
+        // The ordinary call's `RecursionError` check.
+        // SAFETY: this thread's own depth cell.
+        let depth = unsafe { (*depth_cell).get() };
+        if depth >= crate::recursion::recursion_limit() {
+            return None;
+        }
+        if !ops
+            .iter()
+            .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o))
+        {
+            return None;
+        }
+        let first = if matches!(ops[1], Object::Unbound) { 2 } else { 1 };
+        let nargs = ops.len() - first;
+        let mut args: [*const Object; 8] = [std::ptr::null(); 8];
+        for (k, o) in ops[first..].iter().enumerate() {
+            args[k] = o;
+        }
+        for (k, o) in f.defaults[f.defaults.len() - missing..].iter().enumerate() {
+            args[nargs + k] = o;
+        }
+        self.pure_leaf_call(code_rc, f, &args[..nargs + missing], effect, depth)
+    }
+
     /// [`Self::core_pure_call`] for a `CALL_KW` at `pc`: `ops` is the
     /// callee, its self slot, `argc` positionals, the keyword values and
     /// the names tuple. The site's cached `CallPyKwNames` permutation

From e415181be4fe71eb349503a9076f74615e1ca4c9 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 10:43:55 -0700
Subject: [PATCH 097/173] perf: finalize a call's released arguments before the
 caller goes on

CPython runs a __del__ inside the decrement that drops the last
reference, so an argument only a call held dies before the caller's next
instruction. Several paths that finish a call in place released the
call's operands without checking whether a release queued a finalizer,
which then ran at the next back edge, call or return:

- the core loop's frameless leaf call (core_pure_call), its natively
  served constructor or bound-method call, its leaf builtin call, and
  its list.pop() arm now stop for the queued work (after_release!), as
  its other arms that end a binding do;
- a tier-2 native call whose callee's pins (its arguments and locals)
  held an object's last reference now leaves native code after the
  call (the Boxed status), so the finalizer runs before the caller's
  next instruction.

Tier 2 still keeps a loop's own temporaries pinned past their last use,
so a temporary argument it passes is finalized late; the new regression
test covers that case in the interpreter and the frame JIT only.
---
 crates/weavepy-vm/src/lib.rs               |   5 +
 crates/weavepy-vm/src/tier2.rs             |  27 ++++
 tests/regrtest/test_call_release_timing.py | 157 +++++++++++++++++++++
 3 files changed, 189 insertions(+)
 create mode 100644 tests/regrtest/test_call_release_timing.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index d9ee9a44..f51c40c8 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -13695,6 +13695,8 @@ impl Interpreter {
                                     len = start + 1;
                                     last = pc;
                                     pc += 1;
+                                    // (An argument the call held last died.)
+                                    after_release!();
                                     continue;
                                 }
                             }
@@ -13725,6 +13727,7 @@ impl Interpreter {
                                 len = start + 1;
                                 last = pc;
                                 pc += 1;
+                                after_release!();
                                 continue;
                             }
                         }
@@ -13865,6 +13868,7 @@ impl Interpreter {
                                 len -= 1;
                                 last = pc;
                                 pc += 1;
+                                after_release!();
                             }
                             // `append(x)` of a local `append = lst.append` on an
                             // exact list (the site's leaf kind, likewise): the
@@ -13959,6 +13963,7 @@ impl Interpreter {
                                         len = callee_at + 1;
                                         last = pc;
                                         pc += 1;
+                                        after_release!();
                                     }
                                     // As the helper's raise: the operands drop
                                     // wholesale and the pc moves past the call.
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index ff623fa3..b2f13658 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -6323,6 +6323,20 @@ unsafe fn try_native_call(
             }
             child.pins.clear();
             ctx.child = Some(child);
+            // An argument the callee held last died with its pins: the
+            // caller leaves native code after the call, so the queued
+            // finalizer runs before its next instruction, as the
+            // interpreter's own call runs it.
+            if finalizer_queued() {
+                ctx.parked = Some(match expect {
+                    SlotTag::Int | SlotTag::Float | SlotTag::Bool => unpack(bits, expect_tag),
+                    SlotTag::ObjPin if bits != u64::MAX => {
+                        unpack_pins(bits, expect_tag, &ctx.pins)
+                    }
+                    _ => Object::None,
+                });
+                return Some(CallStatus::Boxed as i64);
+            }
             jf.ret_bits = bits;
             jf.ret_tag = expect_tag;
             return Some(CallStatus::Ok as i64);
@@ -6626,6 +6640,19 @@ unsafe fn finish_native_call(
     }
     ctx.child = Some(child);
     ctx.dirty |= child_dirty;
+    // A finalizer the callee's releases queued runs before the caller's
+    // next instruction: the caller leaves native code after the call.
+    if finalizer_queued() {
+        match done {
+            Done::Raised(e) => {
+                ctx.raised = Some(e);
+                return CallStatus::Raised as i64;
+            }
+            Done::Scalar(bits, tag) => ctx.parked = Some(unpack(bits, tag)),
+            Done::Obj(v) => ctx.parked = Some(v),
+        }
+        return CallStatus::Boxed as i64;
+    }
 
     match done {
         Done::Raised(e) => {
diff --git a/tests/regrtest/test_call_release_timing.py b/tests/regrtest/test_call_release_timing.py
new file mode 100644
index 00000000..76f3ba86
--- /dev/null
+++ b/tests/regrtest/test_call_release_timing.py
@@ -0,0 +1,157 @@
+"""A call's released arguments and locals finalize before the caller goes on.
+
+CPython runs a `__del__` inside the decrement that drops the last
+reference: an argument only the call held dies when the call's operands
+are released, and a callee's local dies with its frame, both before the
+caller's next instruction. The call paths that finish a call in place (a
+frameless leaf callee, a store-only constructor, a compiled callee's
+direct return) must run the finalizer just as promptly, in the
+interpreter's core loop, in the frame JIT, and in tier 2.
+"""
+
+import os
+import subprocess
+import sys
+import unittest
+
+N = 3000
+MODE = "--mode"
+
+
+class Tracked:
+    log = []
+
+    def __init__(self, name):
+        self.name = name
+
+    def __del__(self):
+        Tracked.log.append(self.name)
+
+
+class Bare:
+    def __del__(self):
+        Tracked.log.append(None)
+
+
+class Keeps:
+    __slots__ = ("kept",)
+
+    def __init__(self, kept, dropped):
+        self.kept = kept
+
+
+class KeepsDict:
+    def __init__(self, kept, dropped):
+        self.kept = kept
+
+
+def first(kept, dropped):
+    return kept
+
+
+def make(i):
+    t = Tracked(i)
+    return i + 1
+
+
+def leaf_loop(n, log):
+    s = 0
+    for i in range(n):
+        s += first(i, Bare())
+        log.append(len(Tracked.log))
+    return s
+
+
+def slot_ctor_loop(n, log):
+    for i in range(n):
+        k = Keeps(i, Bare())
+        log.append(len(Tracked.log))
+    return k
+
+
+def dict_ctor_loop(n, log):
+    for i in range(n):
+        k = KeepsDict(i, Bare())
+        log.append(len(Tracked.log))
+    return k
+
+
+def local_loop(n, log):
+    s = 0
+    for i in range(n):
+        s += make(i)
+        log.append(len(Tracked.log))
+    return s
+
+
+def builtin_loop(n, log):
+    s = 0
+    for i in range(n):
+        s += len([Bare()])
+        log.append(len(Tracked.log))
+    return s
+
+
+# Tier 2 keeps a loop's temporaries pinned past their last use (the
+# interpreter and the frame JIT release them at once).
+PINS = sys.implementation.name == "weavepy" and MODE not in sys.argv
+
+
+class CallReleaseTimingTest(unittest.TestCase):
+    def setUp(self):
+        Tracked.log.clear()
+
+    def check(self, run, expect_result=None):
+        log = []
+        r = run(N, log)
+        if expect_result is not None:
+            self.assertEqual(r, expect_result)
+        self.assertEqual(log, list(range(1, N + 1)))
+        return r
+
+    @unittest.skipIf(PINS, "tier 2 pins the temporary argument")
+    def test_leaf_call_argument(self):
+        self.check(leaf_loop, sum(range(N)))
+
+    @unittest.skipIf(PINS, "tier 2 pins the temporary argument")
+    def test_slot_constructor_argument(self):
+        self.assertEqual(self.check(slot_ctor_loop).kept, N - 1)
+
+    @unittest.skipIf(PINS, "tier 2 pins the temporary argument")
+    def test_dict_constructor_argument(self):
+        self.assertEqual(self.check(dict_ctor_loop).kept, N - 1)
+
+    @unittest.skipIf(PINS, "tier 2 pins the temporary argument")
+    def test_leaf_builtin_argument(self):
+        self.check(builtin_loop, N)
+
+    def test_callee_local(self):
+        self.check(local_loop, sum(range(1, N + 1)))
+
+    def test_other_tiers(self):
+        if MODE in sys.argv:
+            self.skipTest("already in a mode")
+        modes = {
+            # The frame JIT compiling small bodies at once (tier 2 off).
+            "frame-jit": dict(
+                WEAVEPY_JIT="0",
+                WEAVEPY_FRAME_JIT_TUNE="8,1,3,0",
+                WEAVEPY_FRAME_JIT_HOT="50",
+            ),
+            # The interpreter's core loop alone.
+            "core-loop": dict(WEAVEPY_JIT="0", WEAVEPY_FRAME_JIT="0"),
+        }
+        for name, extra in modes.items():
+            env = dict(os.environ)
+            env.update(extra)
+            r = subprocess.run(
+                [sys.executable, __file__, MODE],
+                env=env,
+                capture_output=True,
+                text=True,
+            )
+            self.assertEqual(r.returncode, 0, name + ": " + r.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main(argv=[a for a in sys.argv if a != MODE])

From 9ac6b260f31be3a02347d618c033d29ba46c737a Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 11:39:23 -0700
Subject: [PATCH 098/173] perf: release an in-place call's operands as one run

The paths that finish a call in place (the core loop's leaf, builtin and
constructor arms, h_call and its leaf and site helpers) released the
call's operands one drop_hot at a time, each copying the value out and
testing the refcount bias again, and checked every operand's
droppability with the global watcher flags reloaded per operand.

They now release the run through release_run (the bias tested once,
each shared owner decremented through its tag and pointer) and test the
flags once (core_all_droppable). mini_interpreter retires 1.2% fewer
instructions; the suite 0.2%.

(Instruction counts measured on e58c3c3, before the rebase onto
649d7f6.)
---
 crates/weavepy-vm/src/frame_jit.rs | 28 +++-------
 crates/weavepy-vm/src/lib.rs       | 86 ++++++++----------------------
 2 files changed, 30 insertions(+), 84 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index b7c01fe8..2577d209 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -4393,9 +4393,7 @@ unsafe fn leaf_site_call(st: &mut State, start: usize, len: usize, missing: usiz
         let Some(r) = (*st.interp).site_leaf_eval(ops, missing, st.depth_cell) else {
             return false;
         };
-        for k in start..len {
-            crate::drop_hot(base.add(k).read());
-        }
+        crate::release_run(base.add(start), len - start);
         base.add(start).write(r);
         st.len = start + 1;
         true
@@ -4911,9 +4909,7 @@ unsafe extern "C" fn h_call(
         }
         Interpreter::core_instance_callee(base.add(start));
         let done = |st: &mut State, r: Object| {
-            for k in start..len {
-                crate::drop_hot(base.add(k).read());
-            }
+            crate::release_run(base.add(start), len - start);
             base.add(start).write(r);
             st.len = start + 1;
             0
@@ -4942,9 +4938,7 @@ unsafe extern "C" fn h_call(
                         None => {}
                         Some(Ok(v)) => return done(st, v),
                         Some(Err(e)) => {
-                            for k in start..len {
-                                crate::drop_hot(base.add(k).read());
-                            }
+                            crate::release_run(base.add(start), len - start);
                             st.len = start;
                             st.pc = pc + 1;
                             st.err = Some(e);
@@ -5110,9 +5104,7 @@ unsafe extern "C" fn h_call(
             None => leaf_call_lane(st, pc, start, len),
             Some(Ok(v)) => done(st, v),
             Some(Err(e)) => {
-                for k in start..len {
-                    crate::drop_hot(base.add(k).read());
-                }
+                crate::release_run(base.add(start), len - start);
                 st.len = start;
                 st.pc = pc + 1;
                 st.err = Some(e);
@@ -5238,9 +5230,7 @@ unsafe extern "C" fn h_call_leaf(
                     let args = std::slice::from_raw_parts(base.add(start + 1), argc + 1);
                     if Rc::strong_count(b) > 1 && Interpreter::core_all_droppable(args) {
                         if let Some(v) = kernel_call(k, args, b) {
-                            for j in start..lenu {
-                                crate::drop_hot(base.add(j).read());
-                            }
+                            crate::release_run(base.add(start), lenu - start);
                             base.add(start).write(v);
                             s.len = start + 1;
                             return 0;
@@ -5293,9 +5283,7 @@ unsafe extern "C" fn h_call_leaf(
                         None
                     };
                     if let Some(r) = r {
-                        for k in start..lenu {
-                            crate::drop_hot(base.add(k).read());
-                        }
+                        crate::release_run(base.add(start), lenu - start);
                         return match r {
                             Ok(v) => {
                                 base.add(start).write(v);
@@ -5417,9 +5405,7 @@ unsafe extern "C" fn h_call_kw(st: *mut State, code: *const CodeObject, pc: u64,
             }
             return RELOAD;
         };
-        for k in start..len {
-            crate::drop_hot(base.add(k).read());
-        }
+        crate::release_run(base.add(start), len - start);
         base.add(start).write(r);
         0
     }
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index f51c40c8..4cb45d02 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -13442,9 +13442,7 @@ impl Interpreter {
                         // SAFETY: every operand was checked to leave by a plain
                         // decrement; the result takes the callee's slot.
                         unsafe {
-                            for k in start..len {
-                                drop_hot(base.add(k).read());
-                            }
+                            release_run(base.add(start), len - start);
                             base.add(start).write(r);
                         }
                         len = start + 1;
@@ -13510,9 +13508,7 @@ impl Interpreter {
                                             // by plain decrements; the result
                                             // takes the callee's slot.
                                             unsafe {
-                                                for k in callee_at..len {
-                                                    drop_hot(base.add(k).read());
-                                                }
+                                                release_run(base.add(callee_at), len - callee_at);
                                                 base.add(callee_at).write(v);
                                             }
                                             len = callee_at + 1;
@@ -13523,9 +13519,7 @@ impl Interpreter {
                                         Some(Err(e)) => {
                                             // SAFETY: as above.
                                             unsafe {
-                                                for k in callee_at..len {
-                                                    drop_hot(base.add(k).read());
-                                                }
+                                                release_run(base.add(callee_at), len - callee_at);
                                             }
                                             len = callee_at;
                                             pc += 1;
@@ -13552,9 +13546,7 @@ impl Interpreter {
                             // SAFETY: every operand was checked droppable;
                             // the result takes the callee's slot.
                             unsafe {
-                                for k in callee_at..len {
-                                    drop_hot(base.add(k).read());
-                                }
+                                release_run(base.add(callee_at), len - callee_at);
                                 base.add(callee_at).write(v);
                             }
                             len = callee_at + 1;
@@ -13586,9 +13578,7 @@ impl Interpreter {
                                         // the step) leave by plain decrements;
                                         // the value takes the callee's slot.
                                         unsafe {
-                                            for k in len - 3..len {
-                                                drop_hot(base.add(k).read());
-                                            }
+                                            release_run(base.add(len - 3), 3);
                                             base.add(len - 3).write(v);
                                         }
                                         len -= 2;
@@ -13601,9 +13591,7 @@ impl Interpreter {
                                     gen_fast::GenNext::Exhausted(v) => {
                                         // SAFETY: as above.
                                         unsafe {
-                                            for k in len - 3..len {
-                                                drop_hot(base.add(k).read());
-                                            }
+                                            release_run(base.add(len - 3), 3);
                                         }
                                         len -= 3;
                                         pc += 1;
@@ -13687,9 +13675,7 @@ impl Interpreter {
                                     // by a plain decrement; the result takes the
                                     // callee's slot.
                                     unsafe {
-                                        for k in start..len {
-                                            drop_hot(base.add(k).read());
-                                        }
+                                        release_run(base.add(start), len - start);
                                         base.add(start).write(r);
                                     }
                                     len = start + 1;
@@ -13719,9 +13705,7 @@ impl Interpreter {
                                 // plain decrement; the result takes the callee's
                                 // slot.
                                 unsafe {
-                                    for k in start..len {
-                                        drop_hot(base.add(k).read());
-                                    }
+                                    release_run(base.add(start), len - start);
                                     base.add(start).write(r);
                                 }
                                 len = start + 1;
@@ -13932,7 +13916,7 @@ impl Interpreter {
                                 let ops = unsafe {
                                     std::slice::from_raw_parts(base.add(first), len - first)
                                 };
-                                if !ops.iter().all(Self::core_droppable) {
+                                if !Self::core_all_droppable(ops) {
                                     break Some(CoreExit::Helper);
                                 }
                                 let r = match kind {
@@ -13955,9 +13939,7 @@ impl Interpreter {
                                         // by plain decrements; the result takes
                                         // the callee's slot.
                                         unsafe {
-                                            for k in callee_at..len {
-                                                drop_hot(base.add(k).read());
-                                            }
+                                            release_run(base.add(callee_at), len - callee_at);
                                             base.add(callee_at).write(v);
                                         }
                                         len = callee_at + 1;
@@ -13970,9 +13952,7 @@ impl Interpreter {
                                     Some(Err(e)) => {
                                         // SAFETY: as above.
                                         unsafe {
-                                            for k in callee_at..len {
-                                                drop_hot(base.add(k).read());
-                                            }
+                                            release_run(base.add(callee_at), len - callee_at);
                                         }
                                         len = callee_at;
                                         pc += 1;
@@ -14815,9 +14795,7 @@ impl Interpreter {
                         // SAFETY: the parts leave the stack (a string's release
                         // runs nothing); the join takes the first slot.
                         unsafe {
-                            for k in len - n..len {
-                                drop_hot(base.add(k).read());
-                            }
+                            release_run(base.add(len - n), n);
                             base.add(len - n).write(joined);
                         }
                         len = len - n + 1;
@@ -17556,9 +17534,7 @@ impl Interpreter {
         }
         // The arguments leave by plain decrements (whatever `__init__`
         // stored holds its own reference).
-        if !frame.stack[self_slot + 1..]
-            .iter()
-            .all(Self::core_droppable)
+        if !Self::core_all_droppable(&frame.stack[self_slot + 1..])
         {
             return false;
         }
@@ -19826,9 +19802,7 @@ impl Interpreter {
                 }
             }
         }
-        if !ops
-            .iter()
-            .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o))
+        if !Self::core_all_droppable(ops)
         {
             return None;
         }
@@ -19890,9 +19864,7 @@ impl Interpreter {
         if depth >= crate::recursion::recursion_limit() {
             return None;
         }
-        if !ops
-            .iter()
-            .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o))
+        if !Self::core_all_droppable(ops)
         {
             return None;
         }
@@ -19953,9 +19925,7 @@ impl Interpreter {
         if unsafe { (*depth_cell).get() } >= crate::recursion::recursion_limit() {
             return None;
         }
-        if !ops
-            .iter()
-            .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o))
+        if !Self::core_all_droppable(ops)
         {
             return None;
         }
@@ -20041,7 +20011,7 @@ impl Interpreter {
             || ck as usize != kwc
             || vals.len() != eff_argc + kwc
             || !fn_is_pure_leaf(f)
-            || !vals.iter().all(Self::core_droppable)
+            || !Self::core_all_droppable(vals)
         {
             return None;
         }
@@ -22084,9 +22054,7 @@ impl Interpreter {
                 return None;
             }
             let args = ops.get(2..2 + argc)?;
-            if !ops
-                .iter()
-                .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o))
+            if !Self::core_all_droppable(ops)
             {
                 return None;
             }
@@ -22105,9 +22073,7 @@ impl Interpreter {
         } else {
             (&[], &[])
         };
-        if !ops
-            .iter()
-            .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o))
+        if !Self::core_all_droppable(ops)
         {
             return None;
         }
@@ -22120,9 +22086,7 @@ impl Interpreter {
     /// for the full handler, which also raises every error.
     #[inline(never)]
     fn core_native_method_kw(ops: &[Object], argc: usize) -> Option<Object> {
-        if !ops
-            .iter()
-            .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o))
+        if !Self::core_all_droppable(ops)
         {
             return None;
         }
@@ -22174,9 +22138,7 @@ impl Interpreter {
         }
         let n = bound.iter().take_while(|a| a.is_some()).count();
         if bound[n..].iter().any(Option::is_some)
-            || !ops
-                .iter()
-                .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o))
+            || !Self::core_all_droppable(ops)
         {
             return None;
         }
@@ -22202,9 +22164,7 @@ impl Interpreter {
             return None;
         }
         let fast = crate::stdlib::datetime_native::bound_fast(&bm.function, &bm.receiver)?;
-        if !ops
-            .iter()
-            .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o))
+        if !Self::core_all_droppable(ops)
         {
             return None;
         }
@@ -66565,7 +66525,7 @@ unsafe fn release_locals(locals: &mut Vec<Object>) {
 /// The `n` values are initialized, owned by the caller, and never read
 /// again.
 #[inline(always)]
-unsafe fn release_run(base: *const Object, n: usize) {
+pub(crate) unsafe fn release_run(base: *const Object, n: usize) {
     // SAFETY: the caller's contract.
     unsafe {
         if !crate::rc::refcounts_biased() {

From f0cfe0e2d349886a6a34964931b8539c95bac150 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 12:06:56 -0700
Subject: [PATCH 099/173] perf: keep a direct call's activation slot at its
 site between calls

A direct call took an activation slot from the interpreter's pool,
bound the callee's code into it (a counted reference), and on its clean
return released the code and pushed the slot back.

The slot of a clean direct return now parks at its call site instead
(DirectSite::act), still holding the callee's code (inline_park_warm).
The site's next call of that code takes it back and binds it without
the pool or the code's reference count; a call of another code releases
the held code and binds the slot as a pooled one. A recursive call
finds the site's slot taken and uses the pool as before, and the site
releases a slot it still holds when it is dropped.

The helper's cost on a method call probe drops by about 20
instructions; tomllib_loads retires 1.6% fewer instructions, deepcopy
0.7% and go 0.6%, the suite 0.3%.

(Instruction counts measured on e58c3c3, before the rebase onto
649d7f6.)
---
 crates/weavepy-vm/src/frame_jit.rs | 58 ++++++++++++++++++++++++++----
 crates/weavepy-vm/src/lib.rs       | 36 ++++++++++++++++---
 2 files changed, 82 insertions(+), 12 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 2577d209..6a04413c 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3991,6 +3991,11 @@ pub(crate) struct DirectSite {
     /// Nonzero while the site has a body or a leaf to take: the native
     /// code tries the direct helper first then.
     hot: usize,
+    /// The activation slot the site's last clean direct return parked
+    /// here (see `Interpreter::inline_park_warm`), still holding its
+    /// callee's code: the next call of that code binds it without taking
+    /// a slot from the pool or a reference to the code.
+    act: Option<Box<crate::InlineAct>>,
 }
 
 /// A [`DirectSite`] key: `code`'s address, with `has_self` in its low bit
@@ -4011,6 +4016,13 @@ impl Drop for DirectSite {
                 unsafe { drop(Rc::from_raw(code as *const CodeObject)) };
             }
         }
+        if let Some(act) = self.act.take() {
+            // SAFETY: a warm slot holds its code's count (see
+            // `Interpreter::inline_park_warm`); parked, its own drop then
+            // treats the field as stale.
+            unsafe { drop(std::ptr::read(&raw const act.frame.code)) };
+            drop(act);
+        }
     }
 }
 
@@ -4523,10 +4535,24 @@ unsafe fn direct_call<const CALL: bool>(
         {
             return None;
         }
-        let act = interp.inline_pool.pop()?;
+        // The site's warm slot, still holding this code (another code's
+        // goes back to being a plain parked slot), or one from the pool.
+        let (act, held) = match site.act.take() {
+            Some(act) if std::ptr::eq(Rc::as_ptr(&act.frame.code), code) => (act, true),
+            Some(act) => {
+                // SAFETY: a warm slot holds its code's count.
+                drop(std::ptr::read(&raw const act.frame.code));
+                (act, false)
+            }
+            None => (interp.inline_pool.pop()?, false),
+        };
         let sw = &mut *st.sw;
         let crate::recursion::Enter::Ok(guard) = crate::recursion::enter_with(sw.depth_cell) else {
-            interp.inline_pool.push(act);
+            if held {
+                site.act = Some(act);
+            } else {
+                interp.inline_pool.push(act);
+            }
             return None;
         };
         // Committed: the caller synced as the core loop's arm syncs it, and
@@ -4536,12 +4562,13 @@ unsafe fn direct_call<const CALL: bool>(
         caller.pc = pc as u32;
         *sw.last = st.last;
         let saved_last = sw.last;
-        let callee = interp
-            .core_bind_plain::<true>(sw, pc, start, has_self, missing, cells, fp, act, guard);
+        let callee = interp.core_bind_plain::<true>(
+            sw, pc, start, has_self, missing, cells, fp, act, guard, held,
+        );
         crate::burst_stats::note_call(crate::burst_stats::CALL_DIRECT);
         let depth = (*sw.inl).len();
         Some(direct_run::<false>(
-            st, pc, start, callee, depth, native, saved_last, nested, binop, 0,
+            st, pc, start, callee, depth, native, saved_last, nested, binop, 0, site,
         ))
     }
 }
@@ -4600,7 +4627,17 @@ unsafe fn run_pushed_directly(
         }
         top.direct = true;
         Some(direct_run::<true>(
-            st, pc, start, callee, depth, native, saved_last, nested, None, from,
+            st,
+            pc,
+            start,
+            callee,
+            depth,
+            native,
+            saved_last,
+            nested,
+            None,
+            from,
+            std::ptr::null_mut(),
         ))
     }
 }
@@ -4609,7 +4646,9 @@ unsafe fn run_pushed_directly(
 /// `depth`, marked `direct`) through its native code from `from`, and
 /// finish its return here: [`direct_call`]'s `Some` results. `PUSHED`
 /// for an activation the core loop's call paths bound (see
-/// [`run_pushed_directly`]), whose cells its clean return checks.
+/// [`run_pushed_directly`]), whose cells its clean return checks; `site`
+/// the direct call's cache otherwise, where its clean return parks the
+/// slot.
 ///
 /// # Safety
 ///
@@ -4627,6 +4666,7 @@ unsafe fn direct_run<const PUSHED: bool>(
     nested: u32,
     binop: Option<(BinOpKind, bool, &Rc<crate::types::TypeObject>)>,
     from: usize,
+    site: *mut DirectSite,
 ) -> u32 {
     // SAFETY: the caller's contract.
     unsafe {
@@ -4718,6 +4758,10 @@ unsafe fn direct_run<const PUSHED: bool>(
                 if done.owns_cells {
                     done.clean = true;
                     interp.inline_park_clean(done);
+                } else if !PUSHED && (*site).act.is_none() {
+                    // The slot stays with its call site, still bound to
+                    // the callee's code, for its next call.
+                    (*site).act = Some(interp.inline_park_warm(done));
                 } else {
                     interp.inline_park_direct(done);
                 }
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 4cb45d02..a7cb112d 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -8289,6 +8289,27 @@ impl Interpreter {
         }
     }
 
+    /// [`Self::inline_park_direct`] for a slot that stays with its direct
+    /// call's site (`frame_jit::DirectSite`) instead of the pool: parked,
+    /// but still holding its code's count, so the site's next call of the
+    /// same code binds it without one.
+    #[inline(always)]
+    pub(crate) fn inline_park_warm(&mut self, mut act: Box<InlineAct>) -> Box<InlineAct> {
+        debug_assert!(
+            act.frame.stack.is_empty()
+                && !act.owns_cells
+                && act.binop.is_none()
+                && !act.discard
+                && act.guard.is_none()
+                && act.init_inst.is_none()
+                && act.act.shell.is_none()
+        );
+        act.parked = true;
+        act.direct = false;
+        act.caller_pending = None;
+        act
+    }
+
     /// `FOR_ITER` over a suspended generator at `pc` of `frame`, as an
     /// inline activation: the generator's boxed frame runs in place (the
     /// `generator_send_lean` shape, without a nested native activation).
@@ -15674,7 +15695,7 @@ impl Interpreter {
         // SAFETY: as above; the checks above hold.
         unsafe {
             self.core_bind_plain::<false>(
-                sw, pc, callee_slot, has_self, missing, false, fp, act, guard,
+                sw, pc, callee_slot, has_self, missing, false, fp, act, guard, false,
             )
         };
         true
@@ -15687,7 +15708,9 @@ impl Interpreter {
         f.defaults.len() >= missing && !f.defaults_maybe_overridden()
     }
 
-    /// The committed half of [`Self::core_call_plain`]: the arguments of
+    /// The committed half of [`Self::core_call_plain`] (`held`: the slot
+    /// is a direct call site's warm one, holding the callee's code): the
+    /// arguments of
     /// the `CALL` at `pc` of `sw`'s running activation (the callable at
     /// `callee_slot`, its self slot filled when `has_self`) move into the
     /// pooled slot `act`'s locals, and `act`, bound to the plain function
@@ -15714,6 +15737,7 @@ impl Interpreter {
         fp: *const crate::object::PyFunction,
         mut act: Box<InlineAct>,
         guard: crate::recursion::Guard,
+        held: bool,
     ) -> *mut Frame {
         // SAFETY: the caller's contract.
         let frame = unsafe { &mut *sw.cur };
@@ -15723,8 +15747,7 @@ impl Interpreter {
         let code_rc: &Rc<CodeObject> = unsafe { &*(*fp).code.as_ptr() };
         // Committed. The arguments move into the slot's (empty, sole-owned)
         // locals; the callable and the empty self slot leave the stack.
-        let code = code_rc.clone();
-        let nlocals = code.varnames.len();
+        let nlocals = code_rc.varnames.len();
         // SAFETY: a parked slot's locals storage is its own, and empty.
         let locals = unsafe { &mut *act.frame.locals.as_ptr() };
         let first = if has_self {
@@ -15765,7 +15788,10 @@ impl Interpreter {
         // while the slot holds `callable`.
         unsafe {
             let fr: &mut Frame = &mut act.frame;
-            std::ptr::write(&raw mut fr.code, code);
+            // (A warm slot, `held`, holds this code already.)
+            if !held {
+                std::ptr::write(&raw mut fr.code, code_rc.clone());
+            }
             std::ptr::write(
                 &raw mut fr.cells,
                 borrowed_rc(crate::object::empty_cells_ref()),

From 1bd2e4fb4f879284c9b51b431109fc5ad9b6aa6f Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 13:42:53 -0700
Subject: [PATCH 100/173] perf: run a compiled callee of CALL_KW directly

A compiled caller's CALL_KW of a plain Python function went through the
core loop's keyword call: the site's shape check (cloning the code), a
fill into fresh locals, the generic inline bind, then the pushed
activation run directly (run_pushed_directly) with its extra checks.

h_call_kw now binds a compiled callee itself when the site's cached
CallPyKwNames permutation holds for the function (the fast half of
kw_names_bind_cached: the same function and code, defaults never
rebound): the keyword values go straight to their slots and the rest to
their defaults (kw_names_fill_locals, as the inline path fills them),
and the activation is bound and run as a direct CALL's is.
core_bind_plain's tail, shared by both, is now core_bind_act. Callees
with cells, *args or **kwargs keep the core loop's path.

A finished activation's locals are also released by release_locals
where the lean frame retires and the slot parks.

go retires 0.8% fewer instructions, closures_decorators 0.7% and
tomllib_loads 0.4%.

(Instruction counts measured on e58c3c3, before the rebase onto
649d7f6.)
---
 crates/weavepy-vm/src/frame_jit.rs          | 124 ++++++++++-
 crates/weavepy-vm/src/lib.rs                | 109 ++++++++--
 tests/regrtest/test_direct_keyword_calls.py | 219 ++++++++++++++++++++
 3 files changed, 432 insertions(+), 20 deletions(-)
 create mode 100644 tests/regrtest/test_direct_keyword_calls.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 6a04413c..51bc0e0f 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -4758,7 +4758,7 @@ unsafe fn direct_run<const PUSHED: bool>(
                 if done.owns_cells {
                     done.clean = true;
                     interp.inline_park_clean(done);
-                } else if !PUSHED && (*site).act.is_none() {
+                } else if !PUSHED && !site.is_null() && (*site).act.is_none() {
                     // The slot stays with its call site, still bound to
                     // the callee's code, for its next call.
                     (*site).act = Some(interp.inline_park_warm(done));
@@ -5382,6 +5382,123 @@ unsafe fn leaf_call_lane(st: &mut State, pc: usize, start: usize, len: usize) ->
     }
 }
 
+/// The `CALL_KW` at `pc` of `code` (its operands from `start` up a
+/// `len`-deep stack: the callee, its self slot, `argc` positionals, `kwc`
+/// keyword values and the names tuple) of a plain function whose body has
+/// native code, called directly as [`direct_call`] calls one: the site's
+/// verified `CallPyKwNames` permutation binds the keywords
+/// (`Interpreter::core_bind_kw_direct`), and the callee's return comes
+/// back here. `None` touched nothing (the core loop's keyword call takes
+/// it, filling the site's slot).
+///
+/// # Safety
+///
+/// As [`direct_call`]'s.
+#[inline(never)]
+unsafe fn direct_call_kw(
+    st: &mut State,
+    code: &CodeObject,
+    pc: usize,
+    start: usize,
+    len: usize,
+    argc: usize,
+    kwc: usize,
+) -> Option<u32> {
+    use weavepy_compiler::InlineCache as IC;
+    // SAFETY: the caller's contract.
+    unsafe {
+        let Object::Function(f) = &*st.stack.add(start) else {
+            return None;
+        };
+        let IC::CallPyKwNames {
+            func_id,
+            perm,
+            argc: ca,
+            kwc: ck,
+        } = code.caches.get(pc as u32)
+        else {
+            return None;
+        };
+        let has_self = !matches!(&*st.stack.add(start + 1), Object::Unbound);
+        let eff_argc = argc + usize::from(has_self);
+        if ca as usize != eff_argc || ck as usize != kwc {
+            return None;
+        }
+        let fp = Rc::as_ptr(f);
+        let code_rc: &Rc<CodeObject> = &*f.code.as_ptr();
+        // The binding the site verified for this function and code (see
+        // `Interpreter::kw_names_bind_cached`, whose slow half fills it).
+        if crate::specialize::rc_id(f) != func_id || f.defaults_maybe_overridden() {
+            return None;
+        }
+        let (covered, _) = crate::code_call_slot(code, pc)?.hit(fp, Rc::as_ptr(code_rc))?;
+        let callee: &CodeObject = code_rc;
+        if callee.is_generator
+            || callee.is_coroutine
+            || callee.is_async_generator
+            || callee.has_varargs
+            || callee.has_varkeywords
+            || !callee.cellvars.is_empty()
+            || !callee.freevars.is_empty()
+            || !f.closure.is_empty()
+            || !Interpreter::lean_code_ok(callee)
+        {
+            return None;
+        }
+        let ext = crate::code_vm_ext(callee)?;
+        let native = ext.frame_jit.get(callee.varnames.len())?;
+        let interp = &mut *st.interp.cast_mut();
+        let nested = interp.direct_calls.get();
+        if nested >= DIRECT_CALL_DEPTH
+            || crate::hot_gates::loop_gen() != st.snap_gen
+            || !native.enters_at(0)
+            || !(*st.sw).calls_ok
+            || (callee.jit_hint.is_compiled()
+                && !callee.jit_hint.is_not_jitable()
+                && !crate::tier2::jit_off_for_process())
+        {
+            return None;
+        }
+        let act = interp.inline_pool.pop()?;
+        let sw = &mut *st.sw;
+        let crate::recursion::Enter::Ok(guard) = crate::recursion::enter_with(sw.depth_cell) else {
+            interp.inline_pool.push(act);
+            return None;
+        };
+        // Committed: the caller synced as the core loop's arm syncs it, and
+        // the callee's activation made the running one.
+        let caller = &mut *st.frame;
+        caller.stack.set_len(len);
+        caller.pc = pc as u32;
+        *sw.last = st.last;
+        let saved_last = sw.last;
+        let cframe = interp.core_bind_kw_direct(
+            sw,
+            pc,
+            has_self,
+            fp,
+            (perm, covered, kwc, argc),
+            act,
+            guard,
+        );
+        crate::burst_stats::note_call(crate::burst_stats::CALL_DIRECT);
+        let depth = (*sw.inl).len();
+        Some(direct_run::<false>(
+            st,
+            pc,
+            start,
+            cframe,
+            depth,
+            native,
+            saved_last,
+            nested,
+            None,
+            0,
+            std::ptr::null_mut(),
+        ))
+    }
+}
+
 /// `CALL_KW` (the `pc`th instruction of `code`) on a stack `len` deep, as
 /// the core loop's arm runs it: a natively served class's constructor, a
 /// native method or builtin that takes keywords, or a pure leaf callee in
@@ -5430,6 +5547,11 @@ unsafe extern "C" fn h_call_kw(st: *mut State, code: *const CodeObject, pc: u64,
             if !matches!(&ops[0], Object::Function(_)) || st.sw.is_null() {
                 return CALL_DECLINED;
             }
+            // A compiled one whose keywords the site's permutation places
+            // runs directly, as `CALL`'s does.
+            if let Some(r) = direct_call_kw(st, code, pc, start, len, argc, kwc) {
+                return r;
+            }
             let frame = &mut *st.frame;
             frame.stack.set_len(len);
             frame.pc = pc as u32;
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index a7cb112d..1d7470c7 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -3988,7 +3988,7 @@ impl Interpreter {
             // bookkeeping), which must not observe a held pool borrow.
             // SAFETY: sole owner (checked above) — no other path can
             // reach this vector, including the drop glue of its elements.
-            unsafe { (*frame.locals.as_ptr()).clear() };
+            unsafe { release_locals(&mut *frame.locals.as_ptr()) };
             let mut pool = self.frame_locals_pool.borrow_mut();
             if pool.len() < POOL_CAP {
                 pool.push(frame.locals.clone());
@@ -8162,21 +8162,8 @@ impl Interpreter {
         if Rc::strong_count(&fr.locals) == 1 {
             // SAFETY: sole owner — no other path reaches this vector.
             let v = unsafe { &mut *fr.locals.as_ptr() };
-            if v.iter().all(|o| {
-                matches!(
-                    o,
-                    Object::Int(_)
-                        | Object::Float(_)
-                        | Object::Bool(_)
-                        | Object::None
-                        | Object::Unbound
-                )
-            }) {
-                // SAFETY: scalars own nothing; no drop glue is skipped.
-                unsafe { v.set_len(0) };
-            } else {
-                v.clear();
-            }
+            // SAFETY: sole owner (checked above).
+            unsafe { release_locals(v) };
         } else {
             // The storage escaped (a materialized frame shares it): the
             // slot takes fresh storage.
@@ -8698,7 +8685,7 @@ impl Interpreter {
                     && Rc::strong_count(&frame.locals) == 1
                 {
                     // SAFETY: sole owner; nothing else reaches the vector.
-                    unsafe { (*frame.locals.as_ptr()).clear() };
+                    unsafe { release_locals(&mut *frame.locals.as_ptr()) };
                     drop(done.guard.take());
                     return Ok(v);
                 }
@@ -10379,7 +10366,7 @@ impl Interpreter {
         }
         if Rc::strong_count(&locals) == 1 {
             // SAFETY: sole owner — no other path reaches this vector.
-            unsafe { (*locals.as_ptr()).clear() };
+            unsafe { release_locals(&mut *locals.as_ptr()) };
             let mut pool = self.frame_locals_pool.borrow_mut();
             if pool.len() < POOL_CAP {
                 pool.push(locals);
@@ -15735,7 +15722,7 @@ impl Interpreter {
         missing: usize,
         cells: bool,
         fp: *const crate::object::PyFunction,
-        mut act: Box<InlineAct>,
+        act: Box<InlineAct>,
         guard: crate::recursion::Guard,
         held: bool,
     ) -> *mut Frame {
@@ -15782,6 +15769,90 @@ impl Interpreter {
             frame.stack.set_len(callee_slot);
             callable
         };
+        // SAFETY: as above.
+        unsafe {
+            self.core_bind_act::<DIRECT>(sw, frame, pc, fp, code_rc, act, guard, held, cells, callable)
+        }
+    }
+
+    /// [`Self::core_bind_plain`] for the `CALL_KW` at `pc` of `sw`'s
+    /// running activation (a direct call's, see `frame_jit::h_call_kw`):
+    /// the site's verified `CallPyKwNames` permutation `perm` places the
+    /// `kwc` keyword values, the `argc` positionals (after the receiver
+    /// when `has_self`) take the first slots, and the parameters `covered`
+    /// doesn't name take their defaults (see `kw_names_fill_locals`).
+    ///
+    /// # Safety
+    ///
+    /// As [`Self::core_bind_plain`]'s, with the names tuple on the stack's
+    /// top and a callee with no cells and no `**kwargs`.
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) unsafe fn core_bind_kw_direct(
+        &mut self,
+        sw: &mut CoreSwitch,
+        pc: usize,
+        has_self: bool,
+        fp: *const crate::object::PyFunction,
+        (perm, covered, kwc, argc): (u32, u32, usize, usize),
+        act: Box<InlineAct>,
+        guard: crate::recursion::Guard,
+    ) -> *mut Frame {
+        // SAFETY: the caller's contract.
+        unsafe {
+            let frame = &mut *sw.cur;
+            let code_rc: &Rc<CodeObject> = &*(*fp).code.as_ptr();
+            // A parked slot's locals storage is its own, and empty.
+            let locals = &mut *act.frame.locals.as_ptr();
+            let names = frame.stack.pop().expect("the names tuple");
+            let items: &[Object] = match &names {
+                Object::Tuple(t) => t,
+                _ => &[],
+            };
+            Self::kw_names_fill_locals(
+                locals,
+                &mut frame.stack,
+                items,
+                &*fp,
+                code_rc,
+                perm,
+                covered,
+                kwc,
+                argc + usize::from(has_self),
+            );
+            drop_hot(names);
+            if !has_self {
+                // The empty self slot owns nothing.
+                let n = frame.stack.len();
+                frame.stack.set_len(n - 1);
+            }
+            let callable = frame.stack.pop().expect("the callee");
+            self.core_bind_act::<true>(sw, frame, pc, fp, code_rc, act, guard, false, false, callable)
+        }
+    }
+
+    /// The common half of [`Self::core_bind_plain`] and
+    /// [`Self::core_bind_kw_direct`]: `act`, its locals filled and
+    /// `callable` off the caller's stack, bound to `fp` and pushed as the
+    /// running activation, the caller (`frame`) waiting past its call.
+    ///
+    /// # Safety
+    ///
+    /// As [`Self::core_bind_plain`]'s.
+    #[inline(always)]
+    #[allow(clippy::too_many_arguments)]
+    unsafe fn core_bind_act<const DIRECT: bool>(
+        &mut self,
+        sw: &mut CoreSwitch,
+        frame: &mut Frame,
+        pc: usize,
+        fp: *const crate::object::PyFunction,
+        code_rc: &Rc<CodeObject>,
+        mut act: Box<InlineAct>,
+        guard: crate::recursion::Guard,
+        held: bool,
+        cells: bool,
+        callable: Object,
+    ) -> *mut Frame {
         frame.pc = pc as u32 + 1;
         // SAFETY: the slot is parked (see `inline_bind_cells`): its handles
         // are stale copies, overwritten without a drop. `fp` stays alive
diff --git a/tests/regrtest/test_direct_keyword_calls.py b/tests/regrtest/test_direct_keyword_calls.py
new file mode 100644
index 00000000..aa5f9eea
--- /dev/null
+++ b/tests/regrtest/test_direct_keyword_calls.py
@@ -0,0 +1,219 @@
+"""Keyword calls between compiled functions that run the callee directly.
+
+A hot `CALL_KW` of a plain Python function whose body is compiled binds
+the keywords through the site's cached permutation and runs the callee
+straight from the caller's native code. Every way such a call can differ
+(defaults and keyword-only defaults, a rebound `__defaults__` or
+`__kwdefaults__`, a method receiver, a raise, a frame the callee looks
+at, a tracer, deep recursion, a finalizer) must behave as the ordinary
+call does.
+"""
+
+import sys
+import unittest
+
+N = 3000
+FORCED = "--forced-frame-jit"
+
+
+def kw(a, b=2, *, c=3, d=4):
+    t = [a, b, c, d]
+    return t[0] * 1000 + t[1] * 100 + t[2] * 10 + t[3]
+
+
+def kw_loop(n):
+    s = 0
+    for i in range(n):
+        s += kw(i, c=5)
+        s += kw(i, b=6)
+        s += kw(a=i, d=7, b=1)
+    return s
+
+
+def expected_kw(n):
+    s = 0
+    for i in range(n):
+        s += i * 1000 + 200 + 50 + 4
+        s += i * 1000 + 600 + 30 + 4
+        s += i * 1000 + 100 + 30 + 7
+    return s
+
+
+class Box:
+    def __init__(self):
+        self.n = 0
+
+    def bump(self, k, *, twice=False):
+        t = [k]
+        self.n += t[0] * (2 if twice else 1)
+        return self.n
+
+
+def method_loop(n):
+    b = Box()
+    for i in range(n):
+        b.bump(i, twice=(i % 2 == 0))
+    return b.n
+
+
+def maybe_raise(a, *, at):
+    t = [a]
+    if a == at:
+        raise ValueError(a)
+    return t[0]
+
+
+def raise_loop(n):
+    s = 0
+    for i in range(n):
+        s += maybe_raise(i, at=n - 7)
+    return s
+
+
+def caller_name(a, *, last):
+    t = [a]
+    if a == last:
+        return sys._getframe(1).f_code.co_name
+    return t[0]
+
+
+def frame_loop(n):
+    r = None
+    for i in range(n):
+        r = caller_name(i, last=n - 3)
+        if isinstance(r, str):
+            return r
+    return r
+
+
+def depth(n, *, step=1):
+    t = [n]
+    if t[0] <= 0:
+        return 0
+    return depth(n - step, step=step) + 1
+
+
+class Tracked:
+    log = []
+
+    def __init__(self, name):
+        self.name = name
+
+    def __del__(self):
+        Tracked.log.append(self.name)
+
+
+def takes(a, *, dropped):
+    t = [a]
+    return t[0]
+
+
+def finalizer_loop(n, log):
+    s = 0
+    for i in range(n):
+        s += takes(i, dropped=Tracked(i))
+        log.append(len(Tracked.log))
+    return s
+
+
+def defaults_loop(n):
+    s = 0
+    for i in range(n):
+        s += kw(i, c=0)
+    return s
+
+
+class DirectKeywordCallTest(unittest.TestCase):
+    def test_forced_frame_jit(self):
+        # The same cases with small bodies compiled by the frame JIT at
+        # once (tier 2 off), so every call above runs directly.
+        if FORCED in sys.argv:
+            self.skipTest("already forced")
+        import os
+        import subprocess
+
+        env = dict(os.environ)
+        env.update(
+            WEAVEPY_JIT="0",
+            WEAVEPY_FRAME_JIT_TUNE="8,1,3,0",
+            WEAVEPY_FRAME_JIT_HOT="50",
+        )
+        r = subprocess.run(
+            [sys.executable, __file__, FORCED],
+            env=env,
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(r.returncode, 0, r.stderr)
+
+    def test_permutations_and_defaults(self):
+        self.assertEqual(kw_loop(N), expected_kw(N))
+
+    def test_method_with_keyword_only(self):
+        expect = sum(i * (2 if i % 2 == 0 else 1) for i in range(N))
+        self.assertEqual(method_loop(N), expect)
+
+    def test_rebound_defaults(self):
+        saved, saved_kw = kw.__defaults__, kw.__kwdefaults__
+        try:
+            self.assertEqual(defaults_loop(N), sum(i * 1000 + 204 for i in range(N)))
+            kw.__defaults__ = (9,)
+            self.assertEqual(defaults_loop(N), sum(i * 1000 + 904 for i in range(N)))
+            kw.__kwdefaults__ = {"c": 3, "d": 8}
+            self.assertEqual(defaults_loop(N), sum(i * 1000 + 908 for i in range(N)))
+        finally:
+            kw.__defaults__, kw.__kwdefaults__ = saved, saved_kw
+        self.assertEqual(defaults_loop(N), sum(i * 1000 + 204 for i in range(N)))
+
+    def test_raise_propagates_with_traceback(self):
+        try:
+            raise_loop(N)
+        except ValueError as e:
+            exc = e
+        else:
+            self.fail("no raise")
+        self.assertEqual(exc.args, (N - 7,))
+        names = []
+        tb = exc.__traceback__
+        while tb is not None:
+            names.append(tb.tb_frame.f_code.co_name)
+            tb = tb.tb_next
+        self.assertEqual(names[-2:], ["raise_loop", "maybe_raise"])
+
+    def test_callee_sees_caller_frame(self):
+        self.assertEqual(frame_loop(N), "frame_loop")
+
+    def test_recursion(self):
+        for _ in range(20):
+            self.assertEqual(depth(200, step=1), 200)
+        with self.assertRaises(RecursionError):
+            depth(sys.getrecursionlimit() + 100, step=1)
+        self.assertEqual(depth(200, step=2), 100)
+
+    def test_finalizer_runs_before_caller_continues(self):
+        if FORCED not in sys.argv and sys.implementation.name == "weavepy":
+            self.skipTest("tier 2 pins the temporary argument")
+        Tracked.log.clear()
+        log = []
+        self.assertEqual(finalizer_loop(N, log), sum(range(N)))
+        self.assertEqual(log, list(range(1, N + 1)))
+
+    def test_tracer_sees_calls(self):
+        kw_loop(N)
+        calls = []
+
+        def tracer(frame, event, arg):
+            if event == "call" and frame.f_code is kw.__code__:
+                calls.append(frame.f_locals["c"])
+            return None
+
+        sys.settrace(tracer)
+        try:
+            kw_loop(5)
+        finally:
+            sys.settrace(None)
+        self.assertEqual(calls, [5, 3, 3] * 5)
+
+
+if __name__ == "__main__":
+    unittest.main(argv=[a for a in sys.argv if a != FORCED])

From 457c284aa3c7cbd86d55c90b40b6322473281745 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 14:43:49 -0700
Subject: [PATCH 101/173] perf: share one frameless leaf call between the call
 paths

Every frameless leaf call (the core loop's, the direct call site's, a
local method's, a fused global call's) inlined pure_leaf_call's tries
bookkeeping and then called pure_leaf_call_inner, which looked up the
callee's extension again and called leaf_site_native, which looked it up
a third time, rechecked the shape and called run_compiled.

pure_leaf_call is now one out-of-line function that every path shares:
it looks the extension up once, passes it to leaf_site_native, and runs
the compiled plan (run_compiled) in line. exceptions retires 1.5% fewer
instructions, mini_interpreter 1.0%, oo_patterns 0.7% and raytrace
0.6%; under callgrind class_creation is unchanged (its +1% here is
layout).

(Instruction counts measured on e58c3c3, before the rebase onto
649d7f6.)
---
 crates/weavepy-vm/src/leaf_plan.rs     | 13 +++---
 crates/weavepy-vm/src/leaf_plan/jit.rs |  2 +-
 crates/weavepy-vm/src/lib.rs           | 56 +++++++++++---------------
 3 files changed, 30 insertions(+), 41 deletions(-)

diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs
index 67bb39e6..5eab94cd 100644
--- a/crates/weavepy-vm/src/leaf_plan.rs
+++ b/crates/weavepy-vm/src/leaf_plan.rs
@@ -994,21 +994,20 @@ impl Interpreter {
     /// code, at recursion depth `depth`, once the callee's warm-up is past
     /// (the frameless call bookkeeping is then only the hit count):
     /// `None` when the plan has no native code, or the callee one of the
-    /// tiny shapes `leaf_eval` answers itself; otherwise the result, or
-    /// `None` inside having done nothing observable.
+    /// tiny shapes `leaf_eval` answers itself (the caller checked `ext`,
+    /// `code`'s extension, for those); otherwise the result, or `None`
+    /// inside having done nothing observable.
     #[cfg(feature = "jit")]
-    #[inline]
+    #[inline(always)]
     pub(crate) fn leaf_site_native<const EFFECT: bool>(
         &self,
         code: &CodeObject,
+        ext: &CodeConstObjects,
         f: &crate::object::PyFunction,
         args: &[*const Object],
         depth: usize,
     ) -> Option<Option<Object>> {
-        let ext = crate::code_vm_ext(code)?;
-        if ext.pure_leaf.load(std::sync::atomic::Ordering::Relaxed) >= 3
-            || code.jit_hint.lean_entries() <= crate::tier2::LEAN_WARM_COMPILE_THRESHOLD_CAP
-        {
+        if code.jit_hint.lean_entries() <= crate::tier2::LEAN_WARM_COMPILE_THRESHOLD_CAP {
             return None;
         }
         let plan = ext.leaf_plan.get()?.as_deref()?;
diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs
index ef6f1365..31342a9a 100644
--- a/crates/weavepy-vm/src/leaf_plan/jit.rs
+++ b/crates/weavepy-vm/src/leaf_plan/jit.rs
@@ -322,7 +322,7 @@ pub(super) fn run<const EFFECT: bool>(
 /// [`run`] for a plan whose native code is already compiled, with the
 /// caller's recursion `depth` in hand: `None` when there is no native
 /// code (yet), otherwise the evaluation's outcome.
-#[inline]
+#[inline(always)]
 pub(super) fn run_compiled<const EFFECT: bool>(
     interp: &Interpreter,
     code: &CodeObject,
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 1d7470c7..64003122 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -20295,8 +20295,9 @@ impl Interpreter {
 
     /// [`Self::pure_leaf_eval`] (with `effect`, of an effect leaf) at
     /// recursion depth `depth`: a warm callee's compiled plan is entered
-    /// directly (see [`Self::leaf_site_native`]).
-    #[inline(always)]
+    /// directly (see [`Self::leaf_site_native`]). The one copy every
+    /// frameless call path shares: its extension is looked up once.
+    #[inline(never)]
     fn pure_leaf_call(
         &self,
         code: &CodeObject,
@@ -20311,7 +20312,26 @@ impl Interpreter {
         if bails >= LEAF_BAIL_LIMIT && bails > done {
             return None;
         }
-        let r = self.pure_leaf_call_inner(code, f, args, effect, depth);
+        let r = 'eval: {
+            // (The tiny shapes `leaf_eval` answers itself skip the probe.)
+            #[cfg(feature = "jit")]
+            if ext.pure_leaf.load(std::sync::atomic::Ordering::Relaxed) < 3 {
+                let fast = if effect {
+                    self.leaf_site_native::<true>(code, ext, f, args, depth)
+                } else {
+                    self.leaf_site_native::<false>(code, ext, f, args, depth)
+                };
+                if let Some(r) = fast {
+                    break 'eval r;
+                }
+            }
+            let _ = depth;
+            if effect {
+                self.pure_leaf_eval::<false, true>(code, f, args)
+            } else {
+                self.pure_leaf_eval::<false, false>(code, f, args)
+            }
+        };
         // (Each half stops counting when full: the counts only steer the
         // decision above.)
         let bump = if r.is_some() { 1 } else { 1 << 32 };
@@ -20322,36 +20342,6 @@ impl Interpreter {
         r
     }
 
-    fn pure_leaf_call_inner(
-        &self,
-        code: &CodeObject,
-        f: &crate::object::PyFunction,
-        args: &[*const Object],
-        effect: bool,
-        depth: usize,
-    ) -> Option<Object> {
-        // (The tiny shapes `leaf_eval` answers itself skip the probe.)
-        #[cfg(feature = "jit")]
-        if code_vm_ext(code)
-            .is_some_and(|e| e.pure_leaf.load(std::sync::atomic::Ordering::Relaxed) < 3)
-        {
-            let fast = if effect {
-                self.leaf_site_native::<true>(code, f, args, depth)
-            } else {
-                self.leaf_site_native::<false>(code, f, args, depth)
-            };
-            if let Some(r) = fast {
-                return r;
-            }
-        }
-        let _ = depth;
-        if effect {
-            self.pure_leaf_eval::<false, true>(code, f, args)
-        } else {
-            self.pure_leaf_eval::<false, false>(code, f, args)
-        }
-    }
-
     /// [`Self::pure_leaf_eval`] for a constructor's leaf `__init__` on the
     /// fresh instance `args[0]` (see `leaf_run`'s `FRESH`): `None` may
     /// leave that instance half-initialized, for the caller to discard.

From d1b8105f92fc1c89d4de93a1ab6c2e1c692d6fbb Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 16:41:02 -0700
Subject: [PATCH 102/173] perf: keep two frameless leaf callees per direct call
 site

A polymorphic call site (an expression tree's node.eval(env) over node
classes, some compiled, some frameless leaves) cached one leaf callee:
two leaves alternating replaced each other on every call, each
replacement paying the miss path's eligibility checks (direct_body,
site_leaf_fits) and a code reference.

The site now keeps a second leaf, as it keeps a second compiled body:
a new leaf takes the first place and the one there moves to the second.
Under callgrind, closures_decorators retires 1.3% fewer instructions and
mini_interpreter 0.9%, with go unchanged. (A four-way cache of bodies
and leaves alike saved more in mini_interpreter but cost go 0.4% in its
hit path.)

(Instruction counts measured on e58c3c3, before the rebase onto
649d7f6.)
---
 crates/weavepy-vm/src/frame_jit.rs            |  30 +++-
 .../regrtest/test_polymorphic_direct_sites.py | 150 ++++++++++++++++++
 2 files changed, 175 insertions(+), 5 deletions(-)
 create mode 100644 tests/regrtest/test_polymorphic_direct_sites.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 51bc0e0f..3ead3978 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3984,10 +3984,14 @@ pub(crate) struct DirectSite {
     alt_key: usize,
     /// A frameless leaf callee the site evaluates in place (see
     /// [`leaf_site_call`]): its code (a counted reference, `0` none), key,
-    /// and defaults taken.
+    /// and defaults taken; and the one before it (a polymorphic method
+    /// call's other receiver class).
     leaf_code: usize,
     leaf_key: usize,
     leaf_missing: u32,
+    alt_leaf_missing: u32,
+    alt_leaf_code: usize,
+    alt_leaf_key: usize,
     /// Nonzero while the site has a body or a leaf to take: the native
     /// code tries the direct helper first then.
     hot: usize,
@@ -4010,7 +4014,7 @@ const DS_RETRY: i32 = std::mem::offset_of!(DirectSite, retry) as i32;
 
 impl Drop for DirectSite {
     fn drop(&mut self) {
-        for code in [self.code, self.alt_code, self.leaf_code] {
+        for code in [self.code, self.alt_code, self.leaf_code, self.alt_leaf_code] {
             if code != 0 {
                 // SAFETY: each holds the count a fill took.
                 unsafe { drop(Rc::from_raw(code as *const CodeObject)) };
@@ -4050,9 +4054,20 @@ impl DirectSite {
     }
 
     /// Record a frameless leaf callee running `code` (with or without its
-    /// self slot), whose last `missing` parameters take defaults.
+    /// self slot), whose last `missing` parameters take defaults, as the
+    /// site's first leaf, the one it held before becoming its second.
     fn fill_leaf(&mut self, code: &Rc<CodeObject>, has_self: bool, missing: usize) {
         let at = Rc::as_ptr(code) as usize;
+        if self.leaf_code != 0 && at != self.leaf_code {
+            let old = std::mem::replace(&mut self.alt_leaf_code, self.leaf_code);
+            self.leaf_code = 0;
+            if old != 0 {
+                // SAFETY: `old` held the count an earlier fill took.
+                unsafe { drop(Rc::from_raw(old as *const CodeObject)) };
+            }
+            self.alt_leaf_key = self.leaf_key;
+            self.alt_leaf_missing = self.leaf_missing;
+        }
         if at != self.leaf_code {
             let old = std::mem::replace(&mut self.leaf_code, Rc::into_raw(code.clone()) as usize);
             if old != 0 {
@@ -4465,10 +4480,15 @@ unsafe fn direct_call<const CALL: bool>(
             (site.alt_native, site.alt_missing as usize)
         } else {
             let nargs = len - start - 2 + usize::from(has_self);
-            let leaf = CALL && site.leaf_key == key;
+            let leaf = CALL && (site.leaf_key == key || site.alt_leaf_key == key);
             if leaf {
                 // The site's frameless leaf, evaluated in place.
-                if leaf_site_call(st, start, len, site.leaf_missing as usize) {
+                let missing = if site.leaf_key == key {
+                    site.leaf_missing
+                } else {
+                    site.alt_leaf_missing
+                };
+                if leaf_site_call(st, start, len, missing as usize) {
                     return Some(0);
                 }
                 // Declined (a leaf that keeps bailing, or one not warm
diff --git a/tests/regrtest/test_polymorphic_direct_sites.py b/tests/regrtest/test_polymorphic_direct_sites.py
new file mode 100644
index 00000000..11c116cf
--- /dev/null
+++ b/tests/regrtest/test_polymorphic_direct_sites.py
@@ -0,0 +1,150 @@
+"""A polymorphic call site's direct-call cache.
+
+A compiled `CALL` site keeps several callees (compiled bodies and
+frameless leaves alike) and replaces them round robin. Whatever the mix,
+and however often it changes, each call must run its own callee with
+its own arguments and defaults, and a callee whose code is swapped or
+whose body compiles later must still run as itself.
+"""
+
+import sys
+import unittest
+
+N = 3000
+FORCED = "--forced-frame-jit"
+
+
+class Num:
+    def __init__(self, v):
+        self.v = v
+
+    def eval(self, env):
+        return self.v
+
+
+class Var:
+    def __init__(self, name):
+        self.name = name
+
+    def eval(self, env):
+        return env[self.name]
+
+
+class Neg:
+    def __init__(self, inner):
+        self.inner = inner
+
+    def eval(self, env):
+        t = [self.inner.eval(env)]
+        return -t[0]
+
+
+class Add:
+    def __init__(self, a, b):
+        self.a = a
+        self.b = b
+
+    def eval(self, env):
+        t = [self.a.eval(env), self.b.eval(env)]
+        return t[0] + t[1]
+
+
+class Scaled:
+    def __init__(self, inner, k=3):
+        self.inner = inner
+        self.k = k
+
+    def eval(self, env, k=None):
+        t = [self.inner.eval(env)]
+        return t[0] * (self.k if k is None else k)
+
+
+class Const:
+    def eval(self, env, bonus=7):
+        return bonus
+
+
+def tree():
+    return Add(
+        Add(Num(1), Var("x")),
+        Add(Neg(Var("y")), Add(Scaled(Num(2)), Add(Const(), Scaled(Var("x"), 5)))),
+    )
+
+
+def run(nodes, env, n):
+    s = 0
+    for i in range(n):
+        node = nodes[i % len(nodes)]
+        s += node.eval(env)
+    return s
+
+
+def expected(nodes, env, n):
+    vals = [node.eval(env) for node in nodes]
+    return sum(vals[i % len(vals)] for i in range(n))
+
+
+class PolymorphicSiteTest(unittest.TestCase):
+    def test_forced_frame_jit(self):
+        if FORCED in sys.argv:
+            self.skipTest("already forced")
+        import os
+        import subprocess
+
+        env = dict(os.environ)
+        env.update(
+            WEAVEPY_JIT="0",
+            WEAVEPY_FRAME_JIT_TUNE="8,1,3,0",
+            WEAVEPY_FRAME_JIT_HOT="50",
+        )
+        r = subprocess.run(
+            [sys.executable, __file__, FORCED],
+            env=env,
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(r.returncode, 0, r.stderr)
+
+    def test_megamorphic_mix(self):
+        env = {"x": 4, "y": 9}
+        t = tree()
+        nodes = [t, Num(5), Var("y"), Neg(Num(3)), Scaled(Var("x")), Const()]
+        self.assertEqual(run(nodes, env, N), expected(nodes, env, N))
+
+    def test_changing_mix(self):
+        env = {"x": 2, "y": 3}
+        groups = [
+            [Num(1), Var("x")],
+            [Neg(Num(2)), Const(), Scaled(Num(1), 4)],
+            [Add(Num(1), Num(2)), Var("y"), Num(9), Neg(Var("x")), Const()],
+        ]
+        for _ in range(3):
+            for nodes in groups:
+                self.assertEqual(run(nodes, env, N // 3), expected(nodes, env, N // 3))
+
+    def test_swapped_code(self):
+        env = {"x": 1, "y": 1}
+        nodes = [Num(1), Var("x"), Neg(Num(1)), Const()]
+        self.assertEqual(run(nodes, env, N), expected(nodes, env, N))
+        saved = Num.eval.__code__
+        try:
+            Num.eval.__code__ = (lambda self, env: self.v * 100).__code__
+            self.assertEqual(run(nodes, env, N), expected(nodes, env, N))
+        finally:
+            Num.eval.__code__ = saved
+        self.assertEqual(run(nodes, env, N), expected(nodes, env, N))
+
+    def test_deep_recursion_through_the_site(self):
+        node = Num(1)
+        for _ in range(150):
+            node = Neg(node)
+        for _ in range(20):
+            self.assertEqual(node.eval({}), 1)
+        for _ in range(sys.getrecursionlimit()):
+            node = Neg(node)
+        with self.assertRaises(RecursionError):
+            node.eval({})
+
+
+if __name__ == "__main__":
+    unittest.main(argv=[a for a in sys.argv if a != FORCED])

From d0f31c28e8235c6b099e996d244626ec8e40ad74 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 18:08:49 -0700
Subject: [PATCH 103/173] perf: prove a slotted constructor's member positions
 once per class version

core_store_init's laid-out path re-read every store's field slot on
each construction to find its member positions, then re-borrowed the
instance's slots and applied the write barrier per store. Its proven
path also cloned the class, the __init__ function and its code.

The positions are now proved once per class version and kept in the
instance plan (InstancePlan::slot_members); the values are built with
the split path's, the write barrier applied once if any needs it, and
written through one borrow of the fresh instance's members. The proven
path holds the plan instead of three handles. raytrace retires 0.8%
fewer instructions.
---
 crates/weavepy-vm/src/lib.rs           |  77 +++++---
 crates/weavepy-vm/src/types.rs         |   4 +
 tests/regrtest/test_slot_store_init.py | 235 +++++++++++++++++++++++++
 3 files changed, 290 insertions(+), 26 deletions(-)
 create mode 100644 tests/regrtest/test_slot_store_init.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 64003122..b2770090 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -17164,15 +17164,17 @@ impl Interpreter {
                     Rc::as_ptr(code),
                 );
                 let missing = Self::missing_defaults(init, code, argc + 1).filter(|_| same)?;
-                Some((ty.clone(), init.clone(), code.clone(), missing))
+                Some((ty.clone(), plan.clone(), missing))
             });
-        if let Some((ty, init, code, missing)) = proven {
+        if let Some((ty, plan, missing)) = proven {
+            let (init, code) = plan.lean_init.as_ref().expect("checked above");
             if self.core_store_init(
                 frame,
                 &ty,
-                &init,
-                &code,
-                None,
+                &plan,
+                init,
+                code,
+                false,
                 argc,
                 missing,
                 self_slot,
@@ -17265,9 +17267,10 @@ impl Interpreter {
             && self.core_store_init(
                 frame,
                 &ty,
+                &plan,
                 init,
                 code,
-                Some(&plan.store_init),
+                true,
                 argc,
                 missing,
                 self_slot,
@@ -17379,16 +17382,18 @@ impl Interpreter {
     /// arguments the stores don't take are released. `false` touches
     /// nothing.
     ///
-    /// `proving` is the class plan's verdict to record when the stores
-    /// prove (or disprove) the shape; `None` when the plan proved it.
+    /// With `prove`, the stores prove (or disprove) the shape, which the
+    /// class's `plan` records; without, the plan proved it. A laid-out
+    /// class's member positions are proved once per plan.
     #[allow(clippy::too_many_arguments)]
     fn core_store_init(
         &self,
         frame: &mut Frame,
         ty: &Rc<TypeObject>,
+        plan: &crate::types::InstancePlan,
         init: &Rc<crate::object::PyFunction>,
         code: &Rc<CodeObject>,
-        proving: Option<&std::sync::atomic::AtomicU8>,
+        prove: bool,
         argc: usize,
         missing: usize,
         self_slot: usize,
@@ -17422,9 +17427,13 @@ impl Interpreter {
         // instances (sites another version recorded say nothing), whose
         // layout positions the stores then fill. A site that recorded
         // anything else for this version never will.
+        let proving = prove.then_some(&plan.store_init);
         let laid_out = ty.slot_layout.get().and_then(Option::as_ref);
         let mut members = [0u8; crate::inst_dict::SHARED_KEYS_CAP];
-        if laid_out.is_some() {
+        let n = shape.stores.len();
+        if let (Some(_), Some(proved)) = (laid_out, plan.slot_members.get()) {
+            members[..n].copy_from_slice(proved);
+        } else if laid_out.is_some() {
             let mut seen = 0u64;
             for (k, &(pc, _)) in shape.stores.iter().enumerate() {
                 match slots.get(pc as usize).map(FieldSlot::get) {
@@ -17447,6 +17456,8 @@ impl Interpreter {
                     _ => return false,
                 }
             }
+            // (The positions hold for this class version: its plan's.)
+            let _ = plan.slot_members.set(members[..n].into());
         } else if let Some(proving) = proving {
             // Every store's site must have last stored the next position
             // of this class version's instances. A site proven for another
@@ -17518,31 +17529,44 @@ impl Interpreter {
                 StoreSrc::Const(c) => consts[c as usize].clone(),
                 StoreSrc::Int(n) => Object::Int(n),
             };
-            if let Some(layout) = laid_out {
-                // The member's place (the write barrier first).
-                i.note_slot_store(&value);
-                // SAFETY: the instance is fresh and unshared; its slots
-                // were laid out over its class's layout at allocation.
-                let place = unsafe { i.slots.peek_mut() }
-                    .and_then(|s| s.values_for_layout_mut(layout))
-                    .and_then(|v| v.get_mut(usize::from(members[k])));
-                match place {
-                    Some(place) => drop(std::mem::replace(place, value)),
-                    None => {
-                        // (Unreachable for a fresh instance; the plain
-                        // store keeps it correct regardless.)
+            split_vals[k].write(value);
+        }
+        if let Some(layout) = laid_out {
+            // SAFETY: the first `n` entries were written above; each is
+            // read exactly once, below.
+            let vals = unsafe {
+                std::slice::from_raw_parts_mut(split_vals.as_mut_ptr().cast::<Object>(), n)
+            };
+            // The write barrier once (`note_slot_store`'s), then each value
+            // into its member's place through one borrow.
+            if !vals.iter().all(Object::is_gc_atomic) {
+                i.ensure_gc_tracked();
+            }
+            // SAFETY: the instance is fresh and unshared; its slots were
+            // laid out over its class's layout at allocation.
+            match unsafe { i.slots.peek_mut() }.and_then(|s| s.values_for_layout_mut(layout)) {
+                Some(places) if members[..n].iter().all(|&m| usize::from(m) < places.len()) => {
+                    for k in 0..n {
+                        // SAFETY: as above.
+                        let value = unsafe { std::ptr::read(&vals[k]) };
+                        drop(std::mem::replace(&mut places[usize::from(members[k])], value));
+                    }
+                }
+                _ => {
+                    // (Unreachable for a fresh instance; the plain stores
+                    // keep it correct regardless.)
+                    for k in 0..n {
+                        // SAFETY: as above.
+                        let value = unsafe { std::ptr::read(&vals[k]) };
                         let at = code.instructions[shape.stores[k].0 as usize].arg;
                         if let Some(name) = code.names.get(at as usize) {
                             i.slot_set(name, value);
                         }
                     }
                 }
-                continue;
             }
-            split_vals[k].write(value);
         }
         if let Some(keys) = keys {
-            let n = shape.stores.len();
             // SAFETY: the first `n` entries were written above; each is
             // read exactly once, by the fill or by the stores after it.
             let vals = unsafe {
@@ -55369,6 +55393,7 @@ impl Interpreter {
             tuple_new,
             bare_alloc,
             store_init: std::sync::atomic::AtomicU8::new(0),
+            slot_members: std::sync::OnceLock::new(),
         }
     }
 
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index b6632c4d..6979a047 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -1086,6 +1086,10 @@ pub struct InstancePlan {
     /// `Interpreter::core_store_init`): `0` not yet proven, `1` proven
     /// for this class version, `2` never.
     pub store_init: std::sync::atomic::AtomicU8,
+    /// A store-only `__init__`'s laid-out member positions, in store
+    /// order, once its stores proved them for this class version (see
+    /// `Interpreter::core_store_init`).
+    pub slot_members: std::sync::OnceLock<Box<[u8]>>,
 }
 
 /// How a fresh instance's `native` payload is provisioned (see
diff --git a/tests/regrtest/test_slot_store_init.py b/tests/regrtest/test_slot_store_init.py
new file mode 100644
index 00000000..cf46cf0b
--- /dev/null
+++ b/tests/regrtest/test_slot_store_init.py
@@ -0,0 +1,235 @@
+"""Constructors of `__slots__` classes whose `__init__` only stores.
+
+A hot `cls(args)` of a class whose instances lay their slots out, with an
+`__init__` that is nothing but `self.name = <parameter or constant>`
+stores, fills the fresh instance's slots straight from the arguments once
+every store's site has proved its slot. Recycled instances keep their
+slot storage for the next instance of the same class. Every way such a
+construction can differ (an unset slot, a parameter stored twice or not
+at all, a default, the instance stored into itself, a class changed after
+warm-up, a subclass's layout, a deleted slot) must behave as the
+ordinary call does.
+"""
+
+import gc
+import sys
+import unittest
+
+N = 3000
+FORCED = "--forced-frame-jit"
+
+
+class V:
+    __slots__ = ("x", "y", "z")
+
+    def __init__(self, x, y, z):
+        self.x = x
+        self.y = y
+        self.z = z
+
+
+class Partial:
+    __slots__ = ("a", "b", "c")
+
+    def __init__(self, a, b):
+        self.b = b
+        self.a = a
+
+
+class Twice:
+    __slots__ = ("p", "q", "tag", "n")
+
+    def __init__(self, v, w=None):
+        self.p = v
+        self.q = v
+        self.tag = "t"
+        self.n = 3
+
+
+class Selfish:
+    __slots__ = ("me", "v", "__weakref__")
+
+    def __init__(self, v):
+        self.me = self
+        self.v = v
+
+
+class Base:
+    __slots__ = ("u", "w")
+
+    def __init__(self, u, w):
+        self.u = u
+        self.w = w
+
+
+class Derived(Base):
+    __slots__ = ("extra",)
+
+
+class Other:
+    __slots__ = ("k", "m")
+
+    def __init__(self):
+        pass
+
+
+class Plain:
+    def __init__(self, x):
+        self.x = x
+
+
+class Changing:
+    __slots__ = ("x", "y")
+
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+
+
+def build_vs(n):
+    out = []
+    for i in range(n):
+        out.append(V(i, i + 1, i + 2))
+    return out
+
+
+def build_partial(n):
+    s = 0
+    for i in range(n):
+        p = Partial(i, -i)
+        s += p.a + p.b
+    return s, p
+
+
+def build_twice(n):
+    for i in range(n):
+        t = Twice([i])
+    return t
+
+
+def build_selfish(n):
+    for i in range(n):
+        s = Selfish(i)
+    return s
+
+
+def build_derived(n):
+    for i in range(n):
+        d = Derived(i, -i)
+    return d
+
+
+def churn(n):
+    s = 0
+    for i in range(n):
+        v = V(i, i, i)
+        s += v.z
+    return s
+
+
+def build_changing(n):
+    for i in range(n):
+        c = Changing(i, i + 1)
+    return c
+
+
+class SlotStoreInitTest(unittest.TestCase):
+    def test_forced_frame_jit(self):
+        # The same cases with small bodies compiled by the frame JIT at
+        # once (tier 2 off).
+        if FORCED in sys.argv:
+            self.skipTest("already forced")
+        import os
+        import subprocess
+
+        env = dict(os.environ)
+        env.update(
+            WEAVEPY_JIT="0",
+            WEAVEPY_FRAME_JIT_TUNE="8,1,3,0",
+            WEAVEPY_FRAME_JIT_HOT="50",
+        )
+        r = subprocess.run(
+            [sys.executable, __file__, FORCED],
+            env=env,
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(r.returncode, 0, r.stderr)
+
+    def test_values(self):
+        vs = build_vs(N)
+        for i, v in enumerate(vs):
+            self.assertEqual((v.x, v.y, v.z), (i, i + 1, i + 2))
+
+    def test_unset_slot(self):
+        s, p = build_partial(N)
+        self.assertEqual(s, 0)
+        self.assertEqual((p.a, p.b), (N - 1, 1 - N))
+        with self.assertRaises(AttributeError):
+            p.c
+
+    def test_shared_parameter_constants_and_default(self):
+        t = build_twice(N)
+        self.assertIs(t.p, t.q)
+        self.assertEqual(t.p, [N - 1])
+        # The list is held by both slots only (and the call's argument).
+        self.assertEqual(sys.getrefcount(t.p), 3)
+        self.assertEqual((t.tag, t.n), ("t", 3))
+
+    def test_instance_stored_into_itself(self):
+        import weakref
+
+        s = build_selfish(N)
+        self.assertIs(s.me, s)
+        self.assertEqual(s.v, N - 1)
+        r = weakref.ref(s)
+        del s
+        gc.collect()
+        self.assertIsNone(r())
+
+    def test_subclass_layout(self):
+        d = build_derived(N)
+        self.assertEqual((d.u, d.w), (N - 1, 1 - N))
+        with self.assertRaises(AttributeError):
+            d.extra
+
+    def test_recycled_instances_start_unset(self):
+        self.assertEqual(churn(N), sum(range(N)))
+        for _ in range(100):
+            o = Other()
+            with self.assertRaises(AttributeError):
+                o.k
+            with self.assertRaises(AttributeError):
+                o.m
+            p = Plain(5)
+            self.assertEqual(p.x, 5)
+            self.assertEqual(vars(p), {"x": 5})
+
+    def test_deleted_slot(self):
+        vs = build_vs(10)
+        del vs[3].y
+        with self.assertRaises(AttributeError):
+            vs[3].y
+        vs[3].y = "back"
+        self.assertEqual(vs[3].y, "back")
+        self.assertEqual(build_vs(N)[-1].y, N)
+
+    def test_class_changed_after_warm_up(self):
+        c = build_changing(N)
+        self.assertEqual((c.x, c.y), (N - 1, N))
+        log = []
+        saved = Changing.x
+        try:
+            Changing.x = property(lambda self: 42, lambda self, v: log.append(v))
+            c = build_changing(5)
+            self.assertEqual(c.x, 42)
+            self.assertEqual(log, [0, 1, 2, 3, 4])
+            self.assertEqual(c.y, 5)
+        finally:
+            Changing.x = saved
+        c = build_changing(N)
+        self.assertEqual((c.x, c.y), (N - 1, N))
+
+
+if __name__ == "__main__":
+    unittest.main(argv=[a for a in sys.argv if a != FORCED])

From e7434f31bae0f48e575075c9b24579cdadc18b63 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 19:43:38 -0700
Subject: [PATCH 104/173] perf: write the stdlib code cache under -B too

`-B`/`PYTHONDONTWRITEBYTECODE` stopped the frozen-module cache from being
written, so every process under it compiled each standard-library module
it imported. Many container images set PYTHONDONTWRITEBYTECODE. With it set,
`import asyncio` retired 1971M instructions and took 1.75s, against 391M
and 0.30s with the cache warm (CPython 3.14: 352M, 0.30s).

The flag governs `.pyc` files beside source files, and CPython ships its
standard library already compiled. The frozen cache lives in the user's
cache directory, keyed by the binary's format, and is the counterpart of
that installed bytecode, so it is now written regardless of the flag. A
user module's `.pyc` is still not written under `-B`, and
`WEAVEPY_FROZEN_CACHE=0` still turns the cache off.
---
 crates/weavepy-vm/src/lib.rs                  | 14 +++--
 .../test_frozen_cache_dont_write_bytecode.py  | 52 +++++++++++++++++++
 2 files changed, 61 insertions(+), 5 deletions(-)
 create mode 100644 tests/regrtest/test_frozen_cache_dont_write_bytecode.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index b2770090..7869bb3d 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -61996,11 +61996,15 @@ impl Interpreter {
         {
             frozen_code_cache::insert(full, &code);
             // RFC 0059 WS5a — persist for future processes as well.
-            // `-B`/`PYTHONDONTWRITEBYTECODE` suppresses the write like
-            // any other bytecode artifact (reads stay enabled).
-            if !self.bytecode_writes_disabled() {
-                frozen_code_cache::write_disk(full, source, &code);
-            }
+            // `-B`/`PYTHONDONTWRITEBYTECODE` doesn't suppress the write:
+            // it governs `.pyc` files beside source files, and CPython
+            // ships its standard library already compiled, so this cache
+            // (in the user's cache directory, keyed by the binary's
+            // format) is the counterpart of that installed bytecode.
+            // Without it, every process under `-B` compiled each stdlib
+            // module it imported (`import asyncio`: about 5x the
+            // instructions). `WEAVEPY_FROZEN_CACHE=0` turns it off.
+            frozen_code_cache::write_disk(full, source, &code);
         }
         self.run_frozen_compiled(full, code, is_package, filename)
     }
diff --git a/tests/regrtest/test_frozen_cache_dont_write_bytecode.py b/tests/regrtest/test_frozen_cache_dont_write_bytecode.py
new file mode 100644
index 00000000..cd0eb020
--- /dev/null
+++ b/tests/regrtest/test_frozen_cache_dont_write_bytecode.py
@@ -0,0 +1,52 @@
+"""The embedded standard library's code cache fills under -B.
+
+`-B`/`PYTHONDONTWRITEBYTECODE` governs `.pyc` files beside source files;
+CPython ships its standard library already compiled. WeavePy's frozen
+cache is the counterpart of that installed bytecode, so a process under
+`-B` still writes it (otherwise every such process compiled each stdlib
+module it imported), while a user module's `.pyc` stays unwritten.
+"""
+
+import os
+import subprocess
+import sys
+import tempfile
+
+
+def cached_names(cache):
+    names = set()
+    for root, _dirs, files in os.walk(cache):
+        names.update(files)
+    return names
+
+
+def run(cache, cwd, code, dont_write):
+    env = dict(os.environ)
+    env['WEAVEPY_FROZEN_CACHE'] = cache
+    env.pop('PYTHONDONTWRITEBYTECODE', None)
+    if dont_write:
+        env['PYTHONDONTWRITEBYTECODE'] = '1'
+    result = subprocess.run([sys.executable, '-c', code], cwd=cwd, env=env,
+                            capture_output=True, text=True, timeout=120)
+    assert result.returncode == 0, (result.stdout, result.stderr)
+    return result.stdout
+
+
+if sys.implementation.name == 'weavepy':
+    with tempfile.TemporaryDirectory() as directory:
+        directory = os.path.realpath(directory)
+        with open(os.path.join(directory, 'frozen_cache_user_mod.py'), 'w') as f:
+            f.write('VALUE = 7\n')
+        cache = os.path.join(directory, 'cache')
+        code = ('import sys, json, frozen_cache_user_mod\n'
+                'assert sys.dont_write_bytecode\n'
+                'print(json.dumps({"v": frozen_cache_user_mod.VALUE}))\n')
+        assert run(cache, directory, code, True) == '{"v": 7}\n'
+        names = cached_names(cache)
+        assert 'json' in names or 'json.decoder' in names, sorted(names)[:20]
+        # The user module's bytecode stays unwritten under -B.
+        assert not os.path.exists(os.path.join(directory, '__pycache__'))
+        # A second process reads what the first wrote.
+        assert run(cache, directory, code, True) == '{"v": 7}\n'
+
+print('frozen cache under -B: ok')

From c31d728fce7b785eaced9562fe26fe695b7a1762 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 12:54:42 -0700
Subject: [PATCH 105/173] perf: sets on the compact dict table

`set` and `frozenset` move from `indexmap::IndexSet` to `SetMap`, the
dict's compact table (`dictmap.rs`) without values: the table is now
generic over its value type (`DictMap` is `Table<Object>`), and
`SetMap` wraps `Table<()>` with the set operations' names. Probes hash
once, compare stored hashes before keys, and take the same `Probe`
types as dicts (`LeafProbe` for the leaf loops' membership tests).

A probe of an empty set still hashes its key, as CPython does, which
also fixes `set().discard(x)` and `set().remove(x)` swallowing a
raising `__hash__` (the tiny-table forcing it replaces ran a user hash
twice on a one-element set).

Measured (instructions per item, macOS, PYTHONHASHSEED=0):
  x in s (int)      345 -> 276   (CPython 332)
  x in s (str)      463 -> 445   (CPython 322)
  s.add(x)          505 -> 405   (CPython 438)
  set(words)        269 -> 196   (CPython 204)
  s1 | s2, s1 & u   629 -> 461   (CPython 443)
Suite geomean 1.440 -> 1.438 (deepcopy -0.7%, tomllib_loads -0.7%).

(Measured on the previous base, 785d1aa, before this branch was
rebuilt on d0f31c2.)
---
 crates/weavepy-vm/src/dictmap.rs      | 608 ++++++++++++++++++++------
 crates/weavepy-vm/src/object.rs       |  60 +--
 crates/weavepy-vm/src/type_surface.rs |   2 +-
 tests/regrtest/test_set_table.py      | 147 +++++++
 4 files changed, 646 insertions(+), 171 deletions(-)
 create mode 100644 tests/regrtest/test_set_table.py

diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index c93f184a..d78ebf60 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -38,10 +38,10 @@ const IX_MASK: u64 = 0xFFFF_FFFF;
 
 /// A stored entry: the key's Python hash, the key, and its value.
 #[derive(Clone, Debug)]
-pub struct Bucket {
+pub struct Bucket<V = Object> {
     pub(crate) hash: i64,
     pub key: DictKey,
-    pub value: Object,
+    pub value: V,
 }
 
 /// A key a table can be probed with: its Python hash and its equality with
@@ -145,10 +145,10 @@ pub fn key_bytes_eq(a: &[u8], b: &[u8]) -> bool {
     }
 }
 
-/// An ordered hash table from [`DictKey`] to [`Object`] (see the module
-/// docs).
-pub struct DictMap {
-    entries: Vec<Bucket>,
+/// An ordered hash table from [`DictKey`] to `V` (see the module docs):
+/// [`DictMap`] for a dict, [`SetMap`] for a set.
+pub struct Table<V> {
+    entries: Vec<Bucket<V>>,
     /// The slot table: empty for a small table, else a power of two long.
     slots: Box<[u64]>,
     /// `64 - log2(slots.len())`: the shift that takes a mixed hash to its
@@ -166,14 +166,14 @@ fn slots_for(n: usize) -> usize {
     (n.max(SMALL + 1) * 3 / 2 + 1).next_power_of_two()
 }
 
-impl Default for DictMap {
+impl<V> Default for Table<V> {
     #[inline]
     fn default() -> Self {
         Self::new()
     }
 }
 
-impl Clone for DictMap {
+impl<V: Clone> Clone for Table<V> {
     fn clone(&self) -> Self {
         Self {
             entries: self.entries.clone(),
@@ -183,7 +183,7 @@ impl Clone for DictMap {
     }
 }
 
-impl fmt::Debug for DictMap {
+impl<V: fmt::Debug> fmt::Debug for Table<V> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_map()
             .entries(self.entries.iter().map(|b| (&b.key, &b.value)))
@@ -191,7 +191,7 @@ impl fmt::Debug for DictMap {
     }
 }
 
-impl DictMap {
+impl<V> Table<V> {
     /// An empty table.
     #[inline]
     pub fn new() -> Self {
@@ -222,9 +222,9 @@ impl DictMap {
     #[inline]
     pub fn from_unique_hashed(
         n: usize,
-        mut entry: impl FnMut(usize) -> (i64, DictKey, Object),
+        mut entry: impl FnMut(usize) -> (i64, DictKey, V),
     ) -> Self {
-        let mut entries: Vec<Bucket> = Vec::with_capacity(n);
+        let mut entries: Vec<Bucket<V>> = Vec::with_capacity(n);
         let dst = entries.as_mut_ptr();
         for i in 0..n {
             let (hash, key, value) = entry(i);
@@ -270,7 +270,7 @@ impl DictMap {
 
     /// The entries in order.
     #[inline]
-    pub fn as_entries(&self) -> &[Bucket] {
+    pub fn as_entries(&self) -> &[Bucket<V>] {
         &self.entries
     }
 
@@ -467,7 +467,7 @@ impl DictMap {
     /// [`Self::get`], always in line: for the interpreter's subscript
     /// fast paths, where the call would cost a fair share of the probe.
     #[inline(always)]
-    pub fn get_hot<Q: Probe + ?Sized>(&self, key: &Q) -> Option<&Object> {
+    pub fn get_hot<Q: Probe + ?Sized>(&self, key: &Q) -> Option<&V> {
         if self.slots.is_empty() {
             if self.entries.is_empty() {
                 return None;
@@ -483,21 +483,21 @@ impl DictMap {
     }
 
     #[inline]
-    pub fn get<Q: Probe + ?Sized>(&self, key: &Q) -> Option<&Object> {
+    pub fn get<Q: Probe + ?Sized>(&self, key: &Q) -> Option<&V> {
         let i = self.get_index_of(key)?;
         // SAFETY: a found position is in bounds.
         Some(unsafe { &self.entries.get_unchecked(i).value })
     }
 
     #[inline]
-    pub fn get_mut<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<&mut Object> {
+    pub fn get_mut<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<&mut V> {
         let i = self.get_index_of(key)?;
         // SAFETY: a found position is in bounds.
         Some(unsafe { &mut self.entries.get_unchecked_mut(i).value })
     }
 
     #[inline]
-    pub fn get_full<Q: Probe + ?Sized>(&self, key: &Q) -> Option<(usize, &DictKey, &Object)> {
+    pub fn get_full<Q: Probe + ?Sized>(&self, key: &Q) -> Option<(usize, &DictKey, &V)> {
         let i = self.get_index_of(key)?;
         let b = &self.entries[i];
         Some((i, &b.key, &b.value))
@@ -507,14 +507,14 @@ impl DictMap {
     pub fn get_full_mut<Q: Probe + ?Sized>(
         &mut self,
         key: &Q,
-    ) -> Option<(usize, &DictKey, &mut Object)> {
+    ) -> Option<(usize, &DictKey, &mut V)> {
         let i = self.get_index_of(key)?;
         let b = &mut self.entries[i];
         Some((i, &b.key, &mut b.value))
     }
 
     #[inline]
-    pub fn get_key_value<Q: Probe + ?Sized>(&self, key: &Q) -> Option<(&DictKey, &Object)> {
+    pub fn get_key_value<Q: Probe + ?Sized>(&self, key: &Q) -> Option<(&DictKey, &V)> {
         let i = self.get_index_of(key)?;
         let b = &self.entries[i];
         Some((&b.key, &b.value))
@@ -563,29 +563,29 @@ impl DictMap {
     // Positional access.
 
     #[inline]
-    pub fn get_index(&self, i: usize) -> Option<(&DictKey, &Object)> {
+    pub fn get_index(&self, i: usize) -> Option<(&DictKey, &V)> {
         self.entries.get(i).map(|b| (&b.key, &b.value))
     }
 
     #[inline]
-    pub fn get_index_mut(&mut self, i: usize) -> Option<(&DictKey, &mut Object)> {
+    pub fn get_index_mut(&mut self, i: usize) -> Option<(&DictKey, &mut V)> {
         self.entries.get_mut(i).map(|b| (&b.key, &mut b.value))
     }
 
     /// Mutable access to a key in place. The caller must keep its hash
     /// and equality unchanged.
     #[inline]
-    pub fn get_index_mut2(&mut self, i: usize) -> Option<(&mut DictKey, &mut Object)> {
+    pub fn get_index_mut2(&mut self, i: usize) -> Option<(&mut DictKey, &mut V)> {
         self.entries.get_mut(i).map(|b| (&mut b.key, &mut b.value))
     }
 
     #[inline]
-    pub fn first(&self) -> Option<(&DictKey, &Object)> {
+    pub fn first(&self) -> Option<(&DictKey, &V)> {
         self.get_index(0)
     }
 
     #[inline]
-    pub fn last(&self) -> Option<(&DictKey, &Object)> {
+    pub fn last(&self) -> Option<(&DictKey, &V)> {
         self.entries.last().map(|b| (&b.key, &b.value))
     }
 
@@ -595,7 +595,7 @@ impl DictMap {
     /// Append `key` (hash `hash`), known to be absent, at slot `pos` (from
     /// a missed [`Self::find_hashed`]): its position.
     #[inline]
-    fn push_at(&mut self, pos: usize, hash: i64, key: DictKey, value: Object) -> usize {
+    fn push_at(&mut self, pos: usize, hash: i64, key: DictKey, value: V) -> usize {
         let i = self.entries.len();
         if self.slots.is_empty() {
             if i < SMALL {
@@ -622,14 +622,14 @@ impl DictMap {
 
     /// Insert or replace: the entry's position and the replaced value.
     #[inline]
-    pub fn insert_full(&mut self, key: DictKey, value: Object) -> (usize, Option<Object>) {
+    pub fn insert_full(&mut self, key: DictKey, value: V) -> (usize, Option<V>) {
         let hash = key.probe_hash();
         self.insert_hashed(hash, key, value)
     }
 
     /// [`Self::insert_full`] with the key's Python hash precomputed.
     #[inline]
-    pub fn insert_hashed(&mut self, hash: i64, key: DictKey, value: Object) -> (usize, Option<Object>) {
+    pub fn insert_hashed(&mut self, hash: i64, key: DictKey, value: V) -> (usize, Option<V>) {
         match self.find_hashed(hash, &key) {
             Ok(i) => {
                 // SAFETY: a found position is in bounds.
@@ -641,13 +641,13 @@ impl DictMap {
     }
 
     #[inline]
-    pub fn insert(&mut self, key: DictKey, value: Object) -> Option<Object> {
+    pub fn insert(&mut self, key: DictKey, value: V) -> Option<V> {
         self.insert_full(key, value).1
     }
 
     /// Append `key` with Python hash `hash` without looking for an equal
     /// key: the caller knows there is none. Its position.
-    pub fn insert_unique_hashed(&mut self, hash: i64, key: DictKey, value: Object) -> usize {
+    pub fn insert_unique_hashed(&mut self, hash: i64, key: DictKey, value: V) -> usize {
         if self.slots.is_empty() {
             return self.push_at(0, hash, key, value);
         }
@@ -661,7 +661,7 @@ impl DictMap {
     }
 
     /// The entry for `key`, to read, change, or fill.
-    pub fn entry(&mut self, key: DictKey) -> Entry<'_> {
+    pub fn entry(&mut self, key: DictKey) -> Entry<'_, V> {
         let hash = key.probe_hash();
         match self.find_hashed(hash, &key) {
             Ok(index) => Entry::Occupied(OccupiedEntry { map: self, index }),
@@ -677,7 +677,7 @@ impl DictMap {
     /// The entry for a probe: its position if present, else a vacancy
     /// that inserts the key the caller builds.
     #[inline(always)]
-    pub fn probe_entry<Q: Probe + ?Sized>(&mut self, key: &Q) -> ProbeEntry<'_> {
+    pub fn probe_entry<Q: Probe + ?Sized>(&mut self, key: &Q) -> ProbeEntry<'_, V> {
         let hash = key.probe_hash();
         match self.find_hashed(hash, key) {
             Ok(index) => ProbeEntry::Occupied(OccupiedEntry { map: self, index }),
@@ -693,7 +693,7 @@ impl DictMap {
     // Removal.
 
     /// Remove entry `i`, shifting the later entries down.
-    pub fn shift_remove_index(&mut self, i: usize) -> Option<(DictKey, Object)> {
+    pub fn shift_remove_index(&mut self, i: usize) -> Option<(DictKey, V)> {
         let n = self.entries.len();
         if i >= n {
             return None;
@@ -725,7 +725,7 @@ impl DictMap {
     }
 
     /// Remove entry `i`, moving the last entry into its place.
-    pub fn swap_remove_index(&mut self, i: usize) -> Option<(DictKey, Object)> {
+    pub fn swap_remove_index(&mut self, i: usize) -> Option<(DictKey, V)> {
         let n = self.entries.len();
         if i >= n {
             return None;
@@ -743,7 +743,7 @@ impl DictMap {
     }
 
     /// Remove the last entry.
-    pub fn pop(&mut self) -> Option<(DictKey, Object)> {
+    pub fn pop(&mut self) -> Option<(DictKey, V)> {
         let n = self.entries.len();
         if n == 0 {
             return None;
@@ -756,34 +756,34 @@ impl DictMap {
         Some((b.key, b.value))
     }
 
-    pub fn shift_remove_full<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(usize, DictKey, Object)> {
+    pub fn shift_remove_full<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(usize, DictKey, V)> {
         let i = self.get_index_of(key)?;
         let (k, v) = self.shift_remove_index(i)?;
         Some((i, k, v))
     }
 
-    pub fn shift_remove_entry<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(DictKey, Object)> {
+    pub fn shift_remove_entry<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(DictKey, V)> {
         let i = self.get_index_of(key)?;
         self.shift_remove_index(i)
     }
 
-    pub fn shift_remove<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<Object> {
+    pub fn shift_remove<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<V> {
         self.shift_remove_entry(key).map(|(_, v)| v)
     }
 
-    pub fn swap_remove<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<Object> {
+    pub fn swap_remove<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<V> {
         let i = self.get_index_of(key)?;
         self.swap_remove_index(i).map(|(_, v)| v)
     }
 
-    pub fn swap_remove_full<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(usize, DictKey, Object)> {
+    pub fn swap_remove_full<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(usize, DictKey, V)> {
         let i = self.get_index_of(key)?;
         let (k, v) = self.swap_remove_index(i)?;
         Some((i, k, v))
     }
 
     /// Keep the entries `keep` accepts, in order.
-    pub fn retain(&mut self, mut keep: impl FnMut(&DictKey, &mut Object) -> bool) {
+    pub fn retain(&mut self, mut keep: impl FnMut(&DictKey, &mut V) -> bool) {
         let n = self.entries.len();
         self.entries.retain_mut(|b| keep(&b.key, &mut b.value));
         if self.entries.len() != n {
@@ -815,8 +815,8 @@ impl DictMap {
     }
 
     /// Remove the entries in `range`, yielding them.
-    pub fn drain<R: std::ops::RangeBounds<usize>>(&mut self, range: R) -> std::vec::IntoIter<(DictKey, Object)> {
-        let out: Vec<(DictKey, Object)> = self.entries.drain(range).map(|b| (b.key, b.value)).collect();
+    pub fn drain<R: std::ops::RangeBounds<usize>>(&mut self, range: R) -> std::vec::IntoIter<(DictKey, V)> {
+        let out: Vec<(DictKey, V)> = self.entries.drain(range).map(|b| (b.key, b.value)).collect();
         self.reindex();
         out.into_iter()
     }
@@ -879,7 +879,7 @@ impl DictMap {
     }
 
     /// Sort the entries with `cmp` (stable).
-    pub fn sort_by(&mut self, mut cmp: impl FnMut(&DictKey, &Object, &DictKey, &Object) -> std::cmp::Ordering) {
+    pub fn sort_by(&mut self, mut cmp: impl FnMut(&DictKey, &V, &DictKey, &V) -> std::cmp::Ordering) {
         self.entries
             .sort_by(|a, b| cmp(&a.key, &a.value, &b.key, &b.value));
         self.reindex();
@@ -895,34 +895,34 @@ impl DictMap {
     // Iteration.
 
     #[inline]
-    pub fn iter(&self) -> Iter<'_> {
+    pub fn iter(&self) -> Iter<'_, V> {
         Iter(self.entries.iter())
     }
 
     #[inline]
-    pub fn iter_mut(&mut self) -> IterMut<'_> {
+    pub fn iter_mut(&mut self) -> IterMut<'_, V> {
         IterMut(self.entries.iter_mut())
     }
 
     /// Mutable keys and values. The caller must keep every key's hash and
     /// equality unchanged.
     #[inline]
-    pub fn iter_mut2(&mut self) -> IterMut2<'_> {
+    pub fn iter_mut2(&mut self) -> IterMut2<'_, V> {
         IterMut2(self.entries.iter_mut())
     }
 
     #[inline]
-    pub fn keys(&self) -> Keys<'_> {
+    pub fn keys(&self) -> Keys<'_, V> {
         Keys(self.entries.iter())
     }
 
     #[inline]
-    pub fn values(&self) -> Values<'_> {
+    pub fn values(&self) -> Values<'_, V> {
         Values(self.entries.iter())
     }
 
     #[inline]
-    pub fn values_mut(&mut self) -> ValuesMut<'_> {
+    pub fn values_mut(&mut self) -> ValuesMut<'_, V> {
         ValuesMut(self.entries.iter_mut())
     }
 
@@ -930,29 +930,29 @@ impl DictMap {
         self.entries.into_iter().map(|b| b.key)
     }
 
-    pub fn into_values(self) -> impl DoubleEndedIterator<Item = Object> + ExactSizeIterator {
+    pub fn into_values(self) -> impl DoubleEndedIterator<Item = V> + ExactSizeIterator {
         self.entries.into_iter().map(|b| b.value)
     }
 }
 
 /// An entry of [`DictMap::entry`].
 #[derive(Debug)]
-pub enum Entry<'a> {
-    Occupied(OccupiedEntry<'a>),
-    Vacant(VacantEntry<'a>),
+pub enum Entry<'a, V = Object> {
+    Occupied(OccupiedEntry<'a, V>),
+    Vacant(VacantEntry<'a, V>),
 }
 
 /// A present entry.
 #[derive(Debug)]
-pub struct OccupiedEntry<'a> {
-    map: &'a mut DictMap,
+pub struct OccupiedEntry<'a, V = Object> {
+    map: &'a mut Table<V>,
     index: usize,
 }
 
 /// An absent key, with the slot it would take.
 #[derive(Debug)]
-pub struct VacantEntry<'a> {
-    map: &'a mut DictMap,
+pub struct VacantEntry<'a, V = Object> {
+    map: &'a mut Table<V>,
     key: DictKey,
     hash: i64,
     pos: usize,
@@ -960,28 +960,28 @@ pub struct VacantEntry<'a> {
 
 /// An entry of [`DictMap::probe_entry`].
 #[derive(Debug)]
-pub enum ProbeEntry<'a> {
-    Occupied(OccupiedEntry<'a>),
-    Vacant(ProbeVacancy<'a>),
+pub enum ProbeEntry<'a, V = Object> {
+    Occupied(OccupiedEntry<'a, V>),
+    Vacant(ProbeVacancy<'a, V>),
 }
 
 /// An absent probe key, with its hash and the slot it would take.
 #[derive(Debug)]
-pub struct ProbeVacancy<'a> {
-    map: &'a mut DictMap,
+pub struct ProbeVacancy<'a, V = Object> {
+    map: &'a mut Table<V>,
     hash: i64,
     pos: usize,
 }
 
-impl<'a> Entry<'a> {
-    pub fn or_insert(self, default: Object) -> &'a mut Object {
+impl<'a, V> Entry<'a, V> {
+    pub fn or_insert(self, default: V) -> &'a mut V {
         match self {
             Entry::Occupied(e) => e.into_mut(),
             Entry::Vacant(e) => e.insert(default),
         }
     }
 
-    pub fn or_insert_with(self, f: impl FnOnce() -> Object) -> &'a mut Object {
+    pub fn or_insert_with(self, f: impl FnOnce() -> V) -> &'a mut V {
         match self {
             Entry::Occupied(e) => e.into_mut(),
             Entry::Vacant(e) => e.insert(f()),
@@ -996,7 +996,7 @@ impl<'a> Entry<'a> {
     }
 }
 
-impl<'a> OccupiedEntry<'a> {
+impl<'a, V> OccupiedEntry<'a, V> {
     #[inline]
     pub fn index(&self) -> usize {
         self.index
@@ -1008,49 +1008,49 @@ impl<'a> OccupiedEntry<'a> {
     }
 
     #[inline]
-    pub fn get(&self) -> &Object {
+    pub fn get(&self) -> &V {
         &self.map.entries[self.index].value
     }
 
     #[inline]
-    pub fn get_mut(&mut self) -> &mut Object {
+    pub fn get_mut(&mut self) -> &mut V {
         &mut self.map.entries[self.index].value
     }
 
     #[inline]
-    pub fn into_mut(self) -> &'a mut Object {
+    pub fn into_mut(self) -> &'a mut V {
         &mut self.map.entries[self.index].value
     }
 
     #[inline]
-    pub fn insert(&mut self, value: Object) -> Object {
+    pub fn insert(&mut self, value: V) -> V {
         std::mem::replace(self.get_mut(), value)
     }
 
-    pub fn shift_remove_entry(self) -> (DictKey, Object) {
+    pub fn shift_remove_entry(self) -> (DictKey, V) {
         self.map
             .shift_remove_index(self.index)
             .expect("occupied entry is present")
     }
 
-    pub fn shift_remove(self) -> Object {
+    pub fn shift_remove(self) -> V {
         self.shift_remove_entry().1
     }
 }
 
-impl<'a> VacantEntry<'a> {
+impl<'a, V> VacantEntry<'a, V> {
     #[inline]
     pub fn key(&self) -> &DictKey {
         &self.key
     }
 
-    pub fn insert(self, value: Object) -> &'a mut Object {
+    pub fn insert(self, value: V) -> &'a mut V {
         let i = self.map.push_at(self.pos, self.hash, self.key, value);
         &mut self.map.entries[i].value
     }
 }
 
-impl<'a> ProbeVacancy<'a> {
+impl<'a, V> ProbeVacancy<'a, V> {
     /// The probe's Python hash.
     #[inline]
     pub fn hash(&self) -> i64 {
@@ -1060,16 +1060,16 @@ impl<'a> ProbeVacancy<'a> {
     /// Insert `key`, which must equal the probe (same hash, same key):
     /// its position.
     #[inline]
-    pub fn insert(self, key: DictKey, value: Object) -> usize {
+    pub fn insert(self, key: DictKey, value: V) -> usize {
         self.map.push_at(self.pos, self.hash, key, value)
     }
 }
 
 macro_rules! entry_iter {
     ($name:ident, $inner:ty, $item:ty, |$b:ident| $map:expr) => {
-        pub struct $name<'a>($inner);
+        pub struct $name<'a, V = Object>($inner);
 
-        impl<'a> Iterator for $name<'a> {
+        impl<'a, V> Iterator for $name<'a, V> {
             type Item = $item;
 
             #[inline]
@@ -1098,7 +1098,7 @@ macro_rules! entry_iter {
             }
         }
 
-        impl DoubleEndedIterator for $name<'_> {
+        impl<V> DoubleEndedIterator for $name<'_, V> {
             #[inline]
             fn next_back(&mut self) -> Option<Self::Item> {
                 self.0.next_back().map(|$b| $map)
@@ -1110,16 +1110,16 @@ macro_rules! entry_iter {
             }
         }
 
-        impl ExactSizeIterator for $name<'_> {
+        impl<V> ExactSizeIterator for $name<'_, V> {
             #[inline]
             fn len(&self) -> usize {
                 self.0.len()
             }
         }
 
-        impl std::iter::FusedIterator for $name<'_> {}
+        impl<V> std::iter::FusedIterator for $name<'_, V> {}
 
-        impl fmt::Debug for $name<'_> {
+        impl<V> fmt::Debug for $name<'_, V> {
             fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
                 f.debug_struct(stringify!($name)).finish_non_exhaustive()
             }
@@ -1127,36 +1127,36 @@ macro_rules! entry_iter {
     };
 }
 
-entry_iter!(Iter, std::slice::Iter<'a, Bucket>, (&'a DictKey, &'a Object), |b| (&b.key, &b.value));
-entry_iter!(IterMut, std::slice::IterMut<'a, Bucket>, (&'a DictKey, &'a mut Object), |b| (&b.key, &mut b.value));
-entry_iter!(IterMut2, std::slice::IterMut<'a, Bucket>, (&'a mut DictKey, &'a mut Object), |b| (&mut b.key, &mut b.value));
-entry_iter!(Keys, std::slice::Iter<'a, Bucket>, &'a DictKey, |b| &b.key);
-entry_iter!(Values, std::slice::Iter<'a, Bucket>, &'a Object, |b| &b.value);
-entry_iter!(ValuesMut, std::slice::IterMut<'a, Bucket>, &'a mut Object, |b| &mut b.value);
+entry_iter!(Iter, std::slice::Iter<'a, Bucket<V>>, (&'a DictKey, &'a V), |b| (&b.key, &b.value));
+entry_iter!(IterMut, std::slice::IterMut<'a, Bucket<V>>, (&'a DictKey, &'a mut V), |b| (&b.key, &mut b.value));
+entry_iter!(IterMut2, std::slice::IterMut<'a, Bucket<V>>, (&'a mut DictKey, &'a mut V), |b| (&mut b.key, &mut b.value));
+entry_iter!(Keys, std::slice::Iter<'a, Bucket<V>>, &'a DictKey, |b| &b.key);
+entry_iter!(Values, std::slice::Iter<'a, Bucket<V>>, &'a V, |b| &b.value);
+entry_iter!(ValuesMut, std::slice::IterMut<'a, Bucket<V>>, &'a mut V, |b| &mut b.value);
 
-impl Clone for Iter<'_> {
+impl<V> Clone for Iter<'_, V> {
     fn clone(&self) -> Self {
         Self(self.0.clone())
     }
 }
 
-impl Clone for Keys<'_> {
+impl<V> Clone for Keys<'_, V> {
     fn clone(&self) -> Self {
         Self(self.0.clone())
     }
 }
 
-impl Clone for Values<'_> {
+impl<V> Clone for Values<'_, V> {
     fn clone(&self) -> Self {
         Self(self.0.clone())
     }
 }
 
 /// The owning iterator of a [`DictMap`].
-pub struct IntoIter(std::vec::IntoIter<Bucket>);
+pub struct IntoIter<V = Object>(std::vec::IntoIter<Bucket<V>>);
 
-impl Iterator for IntoIter {
-    type Item = (DictKey, Object);
+impl<V> Iterator for IntoIter<V> {
+    type Item = (DictKey, V);
 
     #[inline]
     fn next(&mut self) -> Option<Self::Item> {
@@ -1169,50 +1169,50 @@ impl Iterator for IntoIter {
     }
 }
 
-impl DoubleEndedIterator for IntoIter {
+impl<V> DoubleEndedIterator for IntoIter<V> {
     #[inline]
     fn next_back(&mut self) -> Option<Self::Item> {
         self.0.next_back().map(|b| (b.key, b.value))
     }
 }
 
-impl ExactSizeIterator for IntoIter {}
+impl<V> ExactSizeIterator for IntoIter<V> {}
 
-impl fmt::Debug for IntoIter {
+impl<V> fmt::Debug for IntoIter<V> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("IntoIter").finish_non_exhaustive()
     }
 }
 
-impl IntoIterator for DictMap {
-    type Item = (DictKey, Object);
-    type IntoIter = IntoIter;
+impl<V> IntoIterator for Table<V> {
+    type Item = (DictKey, V);
+    type IntoIter = IntoIter<V>;
 
-    fn into_iter(self) -> IntoIter {
+    fn into_iter(self) -> IntoIter<V> {
         IntoIter(self.entries.into_iter())
     }
 }
 
-impl<'a> IntoIterator for &'a DictMap {
-    type Item = (&'a DictKey, &'a Object);
-    type IntoIter = Iter<'a>;
+impl<'a, V> IntoIterator for &'a Table<V> {
+    type Item = (&'a DictKey, &'a V);
+    type IntoIter = Iter<'a, V>;
 
-    fn into_iter(self) -> Iter<'a> {
+    fn into_iter(self) -> Iter<'a, V> {
         self.iter()
     }
 }
 
-impl<'a> IntoIterator for &'a mut DictMap {
-    type Item = (&'a DictKey, &'a mut Object);
-    type IntoIter = IterMut<'a>;
+impl<'a, V> IntoIterator for &'a mut Table<V> {
+    type Item = (&'a DictKey, &'a mut V);
+    type IntoIter = IterMut<'a, V>;
 
-    fn into_iter(self) -> IterMut<'a> {
+    fn into_iter(self) -> IterMut<'a, V> {
         self.iter_mut()
     }
 }
 
-impl Extend<(DictKey, Object)> for DictMap {
-    fn extend<I: IntoIterator<Item = (DictKey, Object)>>(&mut self, iter: I) {
+impl<V> Extend<(DictKey, V)> for Table<V> {
+    fn extend<I: IntoIterator<Item = (DictKey, V)>>(&mut self, iter: I) {
         let iter = iter.into_iter();
         let (lo, _) = iter.size_hint();
         // (As IndexMap: reserve for all when empty, else half, since some
@@ -1224,42 +1224,404 @@ impl Extend<(DictKey, Object)> for DictMap {
     }
 }
 
-impl<'a> Extend<(&'a DictKey, &'a Object)> for DictMap {
-    fn extend<I: IntoIterator<Item = (&'a DictKey, &'a Object)>>(&mut self, iter: I) {
+impl<'a, V: Clone + 'a> Extend<(&'a DictKey, &'a V)> for Table<V> {
+    fn extend<I: IntoIterator<Item = (&'a DictKey, &'a V)>>(&mut self, iter: I) {
         self.extend(iter.into_iter().map(|(k, v)| (k.clone(), v.clone())));
     }
 }
 
-impl FromIterator<(DictKey, Object)> for DictMap {
-    fn from_iter<I: IntoIterator<Item = (DictKey, Object)>>(iter: I) -> Self {
+impl<V> FromIterator<(DictKey, V)> for Table<V> {
+    fn from_iter<I: IntoIterator<Item = (DictKey, V)>>(iter: I) -> Self {
         let mut m = Self::new();
         m.extend(iter);
         m
     }
 }
 
-impl<const N: usize> From<[(DictKey, Object); N]> for DictMap {
-    fn from(arr: [(DictKey, Object); N]) -> Self {
+impl<V, const N: usize> From<[(DictKey, V); N]> for Table<V> {
+    fn from(arr: [(DictKey, V); N]) -> Self {
         Self::from_iter(arr)
     }
 }
 
-impl<Q: Probe + ?Sized> std::ops::Index<&Q> for DictMap {
-    type Output = Object;
+impl<V, Q: Probe + ?Sized> std::ops::Index<&Q> for Table<V> {
+    type Output = V;
 
-    fn index(&self, key: &Q) -> &Object {
+    fn index(&self, key: &Q) -> &V {
         self.get(key).expect("key not found")
     }
 }
 
-impl std::ops::Index<usize> for DictMap {
-    type Output = Object;
+impl<V> std::ops::Index<usize> for Table<V> {
+    type Output = V;
 
-    fn index(&self, i: usize) -> &Object {
+    fn index(&self, i: usize) -> &V {
         &self.entries[i].value
     }
 }
 
+/// The table behind a `dict`.
+pub type DictMap = Table<Object>;
+
+/// The table behind a `set` or `frozenset`: [`Table`]'s entries without
+/// values (a key and its hash), with the set operations' names. Iteration
+/// is in insertion order; a removal moves the last entry into the freed
+/// position (`set` order is arbitrary, and a set's iterator reports any
+/// change of size).
+#[derive(Clone, Default)]
+pub struct SetMap(Table<()>);
+
+impl fmt::Debug for SetMap {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_set().entries(self.iter()).finish()
+    }
+}
+
+/// The keys of a [`SetMap`], in order.
+pub type SetIter<'a> = Keys<'a, ()>;
+
+impl SetMap {
+    #[inline]
+    pub fn new() -> Self {
+        Self(Table::new())
+    }
+
+    #[inline]
+    pub fn with_capacity(n: usize) -> Self {
+        Self(Table::with_capacity(n))
+    }
+
+    /// [`Self::with_capacity`]; the hasher is implied (see [`Table`]).
+    #[inline]
+    pub fn with_capacity_and_hasher(n: usize, _: crate::fasthash::FxBuildHasher) -> Self {
+        Self::with_capacity(n)
+    }
+
+    /// [`Self::new`]; the hasher is implied.
+    #[inline]
+    pub fn with_hasher(_: crate::fasthash::FxBuildHasher) -> Self {
+        Self::new()
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    #[inline]
+    pub fn capacity(&self) -> usize {
+        self.0.capacity()
+    }
+
+    #[inline]
+    pub fn reserve(&mut self, additional: usize) {
+        self.0.reserve(additional);
+    }
+
+    pub fn shrink_to_fit(&mut self) {
+        self.0.shrink_to_fit();
+    }
+
+    #[inline]
+    pub fn clear(&mut self) {
+        self.0.clear();
+    }
+
+    /// The entries in order (each key with its stored hash).
+    #[inline]
+    pub fn as_entries(&self) -> &[Bucket<()>] {
+        self.0.as_entries()
+    }
+
+    /// Add `key`: `false` when an equal key was already present (which
+    /// stays).
+    #[inline]
+    pub fn insert(&mut self, key: DictKey) -> bool {
+        self.insert_full(key).1
+    }
+
+    /// [`Self::insert`], with the key's position.
+    #[inline]
+    pub fn insert_full(&mut self, key: DictKey) -> (usize, bool) {
+        let hash = key.probe_hash();
+        self.insert_hashed(hash, key)
+    }
+
+    /// [`Self::insert_full`] with the key's Python hash precomputed.
+    #[inline]
+    pub fn insert_hashed(&mut self, hash: i64, key: DictKey) -> (usize, bool) {
+        match self.0.find_hashed(hash, &key) {
+            Ok(i) => (i, false),
+            Err(pos) => (self.0.push_at(pos, hash, key, ()), true),
+        }
+    }
+
+    /// Append `key` with Python hash `hash`, known to be absent: its
+    /// position.
+    #[inline]
+    pub fn insert_unique_hashed(&mut self, hash: i64, key: DictKey) -> usize {
+        self.0.insert_unique_hashed(hash, key, ())
+    }
+
+    /// Add a probe's key if absent: the key is built (by `make`) only
+    /// then. `true` when it was added.
+    #[inline]
+    pub fn insert_probe<Q: Probe + ?Sized>(&mut self, key: &Q, make: impl FnOnce() -> DictKey) -> bool {
+        match self.0.probe_entry(key) {
+            ProbeEntry::Occupied(_) => false,
+            ProbeEntry::Vacant(v) => {
+                v.insert(make(), ());
+                true
+            }
+        }
+    }
+
+    /// The position of the key equal to `key`. The probe is hashed even
+    /// when the table is empty (CPython hashes first, so a raising
+    /// `__hash__` surfaces from `x in set()`).
+    #[inline]
+    fn find<Q: Probe + ?Sized>(&self, key: &Q) -> Option<usize> {
+        if self.0.is_empty() {
+            let _ = key.probe_hash();
+            return None;
+        }
+        self.0.get_index_of(key)
+    }
+
+    #[inline]
+    pub fn contains<Q: Probe + ?Sized>(&self, key: &Q) -> bool {
+        self.find(key).is_some()
+    }
+
+    #[inline]
+    pub fn get_index_of<Q: Probe + ?Sized>(&self, key: &Q) -> Option<usize> {
+        self.find(key)
+    }
+
+    /// The stored key equal to `key`.
+    #[inline]
+    pub fn get<Q: Probe + ?Sized>(&self, key: &Q) -> Option<&DictKey> {
+        let i = self.find(key)?;
+        Some(&self.0.as_entries()[i].key)
+    }
+
+    #[inline]
+    pub fn get_full<Q: Probe + ?Sized>(&self, key: &Q) -> Option<(usize, &DictKey)> {
+        let i = self.find(key)?;
+        Some((i, &self.0.as_entries()[i].key))
+    }
+
+    /// The position of the first key with Python hash `hash` that `eq`
+    /// accepts (see [`Table::find_by_hash`]).
+    #[inline]
+    pub fn find_by_hash(&self, hash: i64, eq: impl FnMut(&DictKey) -> bool) -> Option<usize> {
+        self.0.find_by_hash(hash, eq)
+    }
+
+    /// The stored Python hash of the key at `i`.
+    #[inline]
+    pub fn hash_at(&self, i: usize) -> Option<i64> {
+        self.0.hash_at(i)
+    }
+
+    #[inline]
+    pub fn get_index(&self, i: usize) -> Option<&DictKey> {
+        self.0.as_entries().get(i).map(|b| &b.key)
+    }
+
+    #[inline]
+    pub fn first(&self) -> Option<&DictKey> {
+        self.get_index(0)
+    }
+
+    #[inline]
+    pub fn last(&self) -> Option<&DictKey> {
+        self.0.as_entries().last().map(|b| &b.key)
+    }
+
+    /// Remove the key equal to `key` (the last key takes its place):
+    /// whether it was present.
+    #[inline]
+    pub fn swap_remove<Q: Probe + ?Sized>(&mut self, key: &Q) -> bool {
+        self.swap_take(key).is_some()
+    }
+
+    /// [`Self::swap_remove`], returning the removed key.
+    #[inline]
+    pub fn swap_take<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<DictKey> {
+        let i = self.find(key)?;
+        self.0.swap_remove_index(i).map(|(k, ())| k)
+    }
+
+    /// [`Self::swap_take`], with the key's former position.
+    #[inline]
+    pub fn swap_remove_full<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<(usize, DictKey)> {
+        let i = self.find(key)?;
+        self.0.swap_remove_index(i).map(|(k, ())| (i, k))
+    }
+
+    /// Remove the key equal to `key`, shifting the later keys down.
+    #[inline]
+    pub fn shift_remove<Q: Probe + ?Sized>(&mut self, key: &Q) -> bool {
+        self.shift_take(key).is_some()
+    }
+
+    /// [`Self::shift_remove`], returning the removed key.
+    #[inline]
+    pub fn shift_take<Q: Probe + ?Sized>(&mut self, key: &Q) -> Option<DictKey> {
+        let i = self.find(key)?;
+        self.0.shift_remove_index(i).map(|(k, ())| k)
+    }
+
+    #[inline]
+    pub fn swap_remove_index(&mut self, i: usize) -> Option<DictKey> {
+        self.0.swap_remove_index(i).map(|(k, ())| k)
+    }
+
+    #[inline]
+    pub fn shift_remove_index(&mut self, i: usize) -> Option<DictKey> {
+        self.0.shift_remove_index(i).map(|(k, ())| k)
+    }
+
+    /// Remove the last key.
+    #[inline]
+    pub fn pop(&mut self) -> Option<DictKey> {
+        self.0.pop().map(|(k, ())| k)
+    }
+
+    /// Keep the keys `keep` accepts, in order.
+    pub fn retain(&mut self, mut keep: impl FnMut(&DictKey) -> bool) {
+        self.0.retain(|k, ()| keep(k));
+    }
+
+    /// Remove the keys in `range`, yielding them.
+    pub fn drain<R: std::ops::RangeBounds<usize>>(&mut self, range: R) -> impl Iterator<Item = DictKey> {
+        self.0.drain(range).map(|(k, ())| k)
+    }
+
+    /// Sort the keys with `cmp` (stable).
+    pub fn sort_by(&mut self, mut cmp: impl FnMut(&DictKey, &DictKey) -> std::cmp::Ordering) {
+        self.0.sort_by_key_with(|a, b| cmp(a, b));
+    }
+
+    #[inline]
+    pub fn iter(&self) -> SetIter<'_> {
+        self.0.keys()
+    }
+
+    /// The keys in `self` that `other` lacks, in order.
+    pub fn difference<'a>(&'a self, other: &'a SetMap) -> impl Iterator<Item = &'a DictKey> + 'a {
+        self.iter().filter(move |k| !other.contains(*k))
+    }
+
+    /// The keys in both, in `self`'s order.
+    pub fn intersection<'a>(&'a self, other: &'a SetMap) -> impl Iterator<Item = &'a DictKey> + 'a {
+        self.iter().filter(move |k| other.contains(*k))
+    }
+
+    /// `self`'s keys, then `other`'s that `self` lacks.
+    pub fn union<'a>(&'a self, other: &'a SetMap) -> impl Iterator<Item = &'a DictKey> + 'a {
+        self.iter().chain(other.difference(self))
+    }
+
+    /// The keys in exactly one, `self`'s first.
+    pub fn symmetric_difference<'a>(
+        &'a self,
+        other: &'a SetMap,
+    ) -> impl Iterator<Item = &'a DictKey> + 'a {
+        self.difference(other).chain(other.difference(self))
+    }
+
+    pub fn is_subset(&self, other: &SetMap) -> bool {
+        self.len() <= other.len() && self.iter().all(|k| other.contains(k))
+    }
+
+    pub fn is_superset(&self, other: &SetMap) -> bool {
+        other.is_subset(self)
+    }
+
+    pub fn is_disjoint(&self, other: &SetMap) -> bool {
+        let (small, big) = if self.len() <= other.len() {
+            (self, other)
+        } else {
+            (other, self)
+        };
+        small.iter().all(|k| !big.contains(k))
+    }
+}
+
+impl IntoIterator for SetMap {
+    type Item = DictKey;
+    type IntoIter = std::iter::Map<IntoIter<()>, fn((DictKey, ())) -> DictKey>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        fn key((k, ()): (DictKey, ())) -> DictKey {
+            k
+        }
+        self.0.into_iter().map(key as fn((DictKey, ())) -> DictKey)
+    }
+}
+
+impl<'a> IntoIterator for &'a SetMap {
+    type Item = &'a DictKey;
+    type IntoIter = SetIter<'a>;
+
+    fn into_iter(self) -> SetIter<'a> {
+        self.iter()
+    }
+}
+
+impl Extend<DictKey> for SetMap {
+    fn extend<I: IntoIterator<Item = DictKey>>(&mut self, iter: I) {
+        let iter = iter.into_iter();
+        let (lo, _) = iter.size_hint();
+        self.reserve(if self.is_empty() { lo } else { lo.div_ceil(2) });
+        for k in iter {
+            self.insert(k);
+        }
+    }
+}
+
+impl<'a> Extend<&'a DictKey> for SetMap {
+    fn extend<I: IntoIterator<Item = &'a DictKey>>(&mut self, iter: I) {
+        self.extend(iter.into_iter().cloned());
+    }
+}
+
+impl FromIterator<DictKey> for SetMap {
+    fn from_iter<I: IntoIterator<Item = DictKey>>(iter: I) -> Self {
+        let mut s = Self::new();
+        s.extend(iter);
+        s
+    }
+}
+
+impl<const N: usize> From<[DictKey; N]> for SetMap {
+    fn from(arr: [DictKey; N]) -> Self {
+        Self::from_iter(arr)
+    }
+}
+
+/// Equal sizes, and every key of one in the other (order aside).
+impl PartialEq for SetMap {
+    fn eq(&self, other: &Self) -> bool {
+        self.len() == other.len() && self.iter().all(|k| other.contains(k))
+    }
+}
+
+impl std::ops::Index<usize> for SetMap {
+    type Output = DictKey;
+
+    fn index(&self, i: usize) -> &DictKey {
+        &self.0.as_entries()[i].key
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 504fd96f..06a165eb 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -225,9 +225,8 @@ pub enum Object {
     Bytes(SharedSlice<u8>),
     /// Mutable byte string `bytearray(...)`.
     ByteArray(Rc<RefCell<Vec<u8>>>),
-    /// Mutable set `{1, 2, 3}` / `set(...)`. Backed by an
-    /// `IndexSet<DictKey>` so iteration order is insertion order
-    /// (CPython 3.7+ semantics).
+    /// Mutable set `{1, 2, 3}` / `set(...)`. Backed by a
+    /// [`crate::dictmap::SetMap`], a compact hash table of its keys.
     Set(Rc<RefCell<SetData>>),
     /// Immutable set `frozenset(...)`. Wrapped in [`FrozenSetObj`] so the
     /// CPython `so_hash` cache (frozensets are immutable, so their hash is
@@ -2713,9 +2712,9 @@ pub struct SlotDescriptor {
     pub objclass: RefCell<Option<crate::sync::Weak<crate::types::TypeObject>>>,
 }
 
-/// Ordered set backing for [`Object::Set`] and [`Object::FrozenSet`].
-/// Fast-hashed for the same reason as [`DictData`].
-pub type SetData = indexmap::IndexSet<DictKey, crate::fasthash::FxBuildHasher>;
+/// Set backing for [`Object::Set`] and [`Object::FrozenSet`]: the dict
+/// table's layout without values (see [`crate::dictmap::SetMap`]).
+pub type SetData = crate::dictmap::SetMap;
 
 /// Backing object for [`Object::FrozenSet`]: the element storage plus a
 /// lazily-computed hash cache.
@@ -3329,8 +3328,9 @@ pub fn module_class(m: &Rc<PyModule>) -> Option<Rc<crate::types::TypeObject>> {
 }
 
 /// Dictionary key: a hashable [`Object`] wrapped to satisfy the
-/// `Hash + Eq` requirements of `HashMap` / `IndexSet`, and the stored key
-/// type of a dict's [`crate::dictmap::DictMap`].
+/// `Hash + Eq` requirements of `HashMap`, and the stored key type of a
+/// dict's [`crate::dictmap::DictMap`] and a set's
+/// [`crate::dictmap::SetMap`].
 #[derive(Clone, Debug)]
 pub struct DictKey(pub Object);
 
@@ -3618,20 +3618,6 @@ fn meta_hash_is_identity(m: &crate::Rc<crate::types::TypeObject>) -> bool {
             && !m.dunder(crate::types::Dunder::Eq).user_defined())
 }
 
-impl Hash for LeafProbe<'_> {
-    #[inline]
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.hash.hash(state);
-    }
-}
-
-impl indexmap::Equivalent<DictKey> for LeafProbe<'_> {
-    #[inline]
-    fn equivalent(&self, key: &DictKey) -> bool {
-        crate::dictmap::Probe::probe_eq(self, key)
-    }
-}
-
 impl crate::dictmap::Probe for LeafProbe<'_> {
     #[inline(always)]
     fn probe_hash(&self) -> i64 {
@@ -4059,9 +4045,8 @@ impl PartialEq for DictKey {
             }
             // CPython dispatches `__eq__` only between keys in the *same
             // hash bucket*; unequal hashes mean "not equal" without any
-            // user code. A dict's table compares stored hashes first, but
-            // a set's (`indexmap`'s) small-table linear scan compares every
-            // entry through this `eq` with no hash check at all, so enforce
+            // user code. The tables compare stored hashes first, but a
+            // caller comparing two keys directly does not, so enforce
             // the bucket precondition here — sqlalchemy's
             // `constraints.discard(ColumnSet())` probes a 1-element set
             // with a key whose `__eq__` returns a SQL clause that *raises*
@@ -4119,19 +4104,6 @@ thread_local! {
 /// Park `err` as the pending key-comparison error (first one wins, matching
 /// CPython aborting on the first failed compare). Called from the reentrant
 /// `__hash__`/`__eq__` bridges when the callback raises.
-/// `indexmap` skips hashing when probing a table of 0 or 1 entries (it
-/// compares the lone entry directly). CPython always calls `__hash__`
-/// first, so a raising `__hash__` must still surface for `x in {'a'}`
-/// (test_set.test_unhashable_element's `HashError`). Only a user
-/// instance with a Python `__hash__` is affected; the call runs inside a
-/// `key_cmp_scope`, so a raised exception is stashed and re-raised there.
-#[inline]
-pub(crate) fn force_hash_for_tiny_table(len: usize, key: &Object) {
-    if len <= 1 && dict_key_is_reentrant(key) {
-        let _ = py_hash_value(key);
-    }
-}
-
 pub(crate) fn stash_key_cmp_error(err: RuntimeError) {
     KEY_COMPARISON.with(|state| {
         let mut slot = state.error.borrow_mut();
@@ -4146,7 +4118,7 @@ fn key_cmp_error_pending() -> bool {
     KEY_COMPARISON.with(|state| state.error.borrow().is_some())
 }
 
-/// Run `f` — a closure that performs `IndexSet`/`IndexMap` probes which may
+/// Run `f` — a closure that performs set or dict probes which may
 /// re-enter a Python `__hash__`/`__eq__` — and surface any exception those
 /// callbacks raised. The pending error of an *outer* scope is saved across
 /// the call and restored afterwards, so a re-entrant set/dict mutation from
@@ -11026,17 +10998,11 @@ impl Object {
                 let key = crate::builtins::set_membership_key(item)?;
                 // A custom `__eq__` raising during a hash collision aborts the
                 // lookup in CPython (test_badcmp `s.__contains__(BadCmp())`).
-                key_cmp_scope(|| {
-                    force_hash_for_tiny_table(s.borrow().len(), &key.0);
-                    s.borrow().contains(&key)
-                })
+                key_cmp_scope(|| s.borrow().contains(&key))
             }
             Object::FrozenSet(s) => {
                 let key = crate::builtins::set_membership_key(item)?;
-                key_cmp_scope(|| {
-                    force_hash_for_tiny_table(s.len(), &key.0);
-                    s.contains(&key)
-                })
+                key_cmp_scope(|| s.contains(&key))
             }
             Object::Bytes(haystack) => bytes_membership(haystack, item),
             Object::ByteArray(haystack) => {
diff --git a/crates/weavepy-vm/src/type_surface.rs b/crates/weavepy-vm/src/type_surface.rs
index 6da30d77..7835d39f 100644
--- a/crates/weavepy-vm/src/type_surface.rs
+++ b/crates/weavepy-vm/src/type_surface.rs
@@ -1608,7 +1608,7 @@ fn set_like(model: &Object, items: Vec<DictKey>) -> Object {
 /// bpo-46615 regression tests, can re-enter and `clear()` the operands)
 /// *without* holding a borrow on any live `Object::Set` cell. We snapshot
 /// the elements out under a short-lived borrow (`set_items`) and rebuild a
-/// detached `IndexSet`, so a re-entrant mutation touches a cell we are no
+/// detached set table, so a re-entrant mutation touches a cell we are no
 /// longer borrowing instead of panicking with `BorrowMutError`.
 fn snapshot_set(o: &Object) -> crate::object::SetData {
     let mut out = crate::object::SetData::default();
diff --git a/tests/regrtest/test_set_table.py b/tests/regrtest/test_set_table.py
new file mode 100644
index 00000000..4dd0ddb9
--- /dev/null
+++ b/tests/regrtest/test_set_table.py
@@ -0,0 +1,147 @@
+"""Set and frozenset tables: equal numbers share a slot, NaN finds itself,
+user hashes run once per probe, errors surface, and iteration notices a
+change of size, across the small (scanned) and slotted table sizes."""
+
+
+class Hashed:
+    calls = 0
+
+    def __init__(self, value, h=None):
+        self.value = value
+        self.h = value if h is None else h
+
+    def __hash__(self):
+        Hashed.calls += 1
+        return self.h
+
+    def __eq__(self, other):
+        return isinstance(other, Hashed) and self.value == other.value
+
+
+class BadHash:
+    def __hash__(self):
+        raise ZeroDivisionError("bad hash")
+
+
+class BadEq:
+    def __hash__(self):
+        return 10**9 + 7
+
+    def __eq__(self, other):
+        raise ValueError("bad eq")
+
+
+for size in (0, 1, 7, 8, 9, 40, 1000):
+    base = list(range(size))
+    for factory in (set, frozenset):
+        s = factory(base)
+        assert len(s) == size
+        # 1 == 1.0 == True, and -0.0 == 0: one slot each.
+        for i in base:
+            assert i in s and float(i) in s
+        if size > 1:
+            assert True in s and 1.0 in s and -0.0 in s and 0.0 in s
+        assert (size + 0.5) not in s
+        assert "0" not in s
+        # Unhashable probes raise; a set probe retries as a frozenset.
+        try:
+            [] in s
+        except TypeError:
+            pass
+        else:
+            raise AssertionError("unhashable list probe did not raise")
+        assert set() not in s
+
+    m = set(base)
+    m.add(True)
+    m.add(1.0)
+    assert len(m) == max(size, 2) if size else len(m) == 1
+    if size > 1:
+        # The first-inserted key stays.
+        assert type([x for x in m if x == 1][0]) is int
+
+    # NaN: identity finds the stored object; another NaN is a new key.
+    nan = float("nan")
+    m = set(base)
+    m.add(nan)
+    assert nan in m
+    assert float("nan") not in m
+    m.add(float("nan"))
+    assert len(m) == size + 2
+    m.discard(nan)
+    assert nan not in m and len(m) == size + 1
+
+    # A user hash runs once per probe, and once per insert.
+    m = set(base)
+    Hashed.calls = 0
+    k = Hashed(3)
+    m.add(k)
+    assert Hashed.calls == 1, Hashed.calls
+    assert Hashed(3) in m
+    assert Hashed.calls == 2, Hashed.calls
+    assert Hashed(4, 3) not in m
+    assert Hashed.calls == 3, Hashed.calls
+    m.discard(Hashed(3))
+    assert Hashed.calls == 4 and k not in m
+
+    # A raising hash surfaces from every operation, even on an empty set.
+    for op in (lambda t: BadHash() in t, lambda t: t.add(BadHash()),
+               lambda t: t.discard(BadHash())):
+        try:
+            op(set(base))
+        except ZeroDivisionError:
+            pass
+        else:
+            raise AssertionError("hash error swallowed")
+
+    # A raising __eq__ surfaces only when hashes collide.
+    m = set(base)
+    m.add(BadEq())
+    try:
+        BadEq() in m
+    except ValueError:
+        pass
+    else:
+        raise AssertionError("eq error swallowed")
+    assert Hashed(77, 10**9 + 8) not in m
+
+    # Changing size during iteration raises; same-size churn does not crash.
+    m = set(base) | {"x"}
+    try:
+        for v in m:
+            m.add(object())
+    except RuntimeError as e:
+        assert "changed size during iteration" in str(e), e
+    else:
+        raise AssertionError("set mutation during iteration not detected")
+    m = set(base) | {"x"}
+    seen = 0
+    for v in m:
+        m.discard(v)
+        m.add(v)
+        seen += 1
+        if seen > 3 * len(m) + 3:
+            break
+
+    # Removals keep every survivor findable.
+    m = set(range(size))
+    for i in range(0, size, 3):
+        m.remove(i)
+    assert m == {i for i in range(size) if i % 3}
+    assert all(i in m for i in range(size) if i % 3)
+    while m:
+        x = m.pop()
+        assert x not in m
+
+# Set algebra and comparisons across sizes and types.
+a = set(range(0, 60, 2))
+b = frozenset(range(0, 60, 3))
+assert a & b == {x for x in range(60) if x % 6 == 0}
+assert a | b == {x for x in range(60) if x % 2 == 0 or x % 3 == 0}
+assert a - b == {x for x in range(0, 60, 2) if x % 3}
+assert a ^ b == (a | b) - (a & b)
+assert frozenset(a) == a and a == frozenset(a)
+assert hash(frozenset(a)) == hash(frozenset(sorted(a, reverse=True)))
+assert {1, 2} < {1, 2, 3} and not {1, 2} < {1, 2}
+assert {1.0, 2} == {True, 2.0}
+print("set table ok")

From 8ac16fda47671609115eaba6eb7ad7bc93a94d25 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 13:21:00 -0700
Subject: [PATCH 106/173] perf: step builtin adapters natively in every
 FOR_ITER

`lazy_core_next`, the pure step every `FOR_ITER` takes first (the core
loop's, the frame JIT's `h_for_iter`, tier 2's), now dispatches on the
adapter's kind once and steps more of them without leaving native code:

- `map` and `filter` with any callback whose call runs no Python
  (`pure_callback`): a warm pure-leaf function or a bound method over
  one, a registered leaf builtin or a leaf method bound to its builtin
  receiver, or a builtin type's native constructor (`int`, `str`,
  `type`, ...); `map` over two cursors too.
- `enumerate` over such an adapter (`enumerate(map(f, xs))`).
- `itertools.groupby` and its groupers when the key function is pure
  and the key comparisons are native.

The frame JIT's `FOR_ITER` no longer exits to the core loop for an
adapter whose step runs Python code (a `map` over a Python function):
it runs the core loop's lane in place, the frame synced and published,
and continues natively. `seq_call1` (the general step's callback, and
`list(map(...))`'s) takes the pure callbacks too, and binds a bound
method over a Python function directly.

A callback's `StopIteration` now ends the adapter's iteration, as in
CPython (`for x in map(f, xs)` used to propagate it).

Measured (instructions per item, macOS; CPython in parentheses):
  for x in map(int, xs)            2527 -> 520   (335)
  for x in map(len, lists)         1930 -> 750   (399)
  for x in map(obj.method, xs)     2006 -> 660   (838)
  for x in map(lambda a, b: a + b, xs, ys)  2487 -> 1113 (845)
  for i, x in enumerate(map(f, xs)) 2663 -> 1442 (916)
  groupby(xs, key) + grouper loop  2997 -> 1276 (1005)
  groupby(keys) + grouper loop     2422 -> 804   (458)
  sum(map(f, xs))                  1139 -> 871   (558)
  statistics.mean (per call / 100) 15161 -> 13955 (5309)
Suite geomean 1.438 -> 1.437 (stdlib_utils -0.6%, tomllib_loads -0.6%).

(Measured on the previous base, 785d1aa, before this branch was
rebuilt on d0f31c2.)
---
 crates/weavepy-vm/src/frame_jit.rs      |  66 +++++-
 crates/weavepy-vm/src/lib.rs            |  42 +++-
 crates/weavepy-vm/src/seqiter.rs        | 292 ++++++++++++++++++++----
 tests/regrtest/test_adapter_for_iter.py | 196 ++++++++++++++++
 4 files changed, 530 insertions(+), 66 deletions(-)
 create mode 100644 tests/regrtest/test_adapter_for_iter.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 3ead3978..8d24576a 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1381,19 +1381,20 @@ unsafe extern "C" fn h_for_iter_rest(
                 None => 3,
             };
         }
-        // A native adapter whose step runs no code (`zip` of native
-        // iterators, `itertools.repeat`, ...); its exhaustion is the
-        // interpreter's.
+        // A native adapter: a step that runs no code (`zip` of native
+        // iterators, `map` with a pure callback, ...) in place; any other
+        // (a Python callback, exhaustion) through the core loop's lane.
         Object::LazyIter(l) => {
             // SAFETY: the code passes its live state, whose interpreter
             // is dormant while the helper runs.
             let interp = unsafe { &*st.interp };
-            let Some(Some(v)) = interp.lazy_core_next(l) else {
-                return 3;
-            };
-            // SAFETY: the slot above the iterator is free.
-            unsafe { out.write(v) };
-            return 1;
+            if let Some(Some(v)) = interp.lazy_core_next(l) {
+                // SAFETY: the slot above the iterator is free.
+                unsafe { out.write(v) };
+                return 1;
+            }
+            // SAFETY: as above.
+            return unsafe { for_iter_lane(st, it, pc) };
         }
         _ => return 3,
     };
@@ -1569,6 +1570,53 @@ unsafe extern "C" fn h_for_iter_rest(
     1
 }
 
+/// The `FOR_ITER` at `pc` over the native adapter at `it` (the stack's top)
+/// whose step may run Python code (a `map` over a Python function), as the
+/// core loop's lane runs it (`Interpreter::core_for_iter_lane`): the frame
+/// synced and waiting on the pending list meanwhile. [`h_for_iter`]'s
+/// results: `1` the item above the iterator, `2` the exhausted iterator
+/// retired, `FOR_SWITCHED` anything else for the core loop (a raise, a
+/// pending stop; the frame synced past the instruction), `3` nothing done.
+///
+/// # Safety
+///
+/// As [`h_for_iter_rest`]'s.
+#[inline(never)]
+unsafe fn for_iter_lane(st: &mut State, it: *mut Object, pc: u64) -> u32 {
+    // SAFETY: the caller's contract.
+    unsafe {
+        if st.sw.is_null() || !std::ptr::eq((*st.sw).cur, st.frame) {
+            return 3;
+        }
+        let pc = pc as usize;
+        let slot = it.offset_from(st.stack) as usize;
+        let frame = &mut *st.frame;
+        frame.stack.set_len(slot + 1);
+        frame.pc = pc as u32;
+        let sw = &mut *st.sw;
+        *sw.last = st.last;
+        let interp = &mut *st.interp.cast_mut();
+        if !interp.core_for_iter_lane(sw) {
+            return 3;
+        }
+        let frame = &*st.frame;
+        if sw.pending.is_none()
+            && std::ptr::eq(sw.cur, st.frame)
+            && crate::hot_gates::loop_gen() == st.snap_gen
+        {
+            let at = frame.pc as usize;
+            let depth = frame.stack.len();
+            if at == pc + 1 && depth == slot + 2 {
+                return 1;
+            }
+            if depth == slot && at == for_iter_exit(&frame.code, pc) {
+                return 2;
+            }
+        }
+        FOR_SWITCHED
+    }
+}
+
 /// [`h_for_iter`] switched to a generator's activation (`RELOAD`).
 const FOR_SWITCHED: u32 = 4;
 /// [`h_for_iter`]'s iterator raised (`RAISED`, the state past it).
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 7869bb3d..861a5665 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -38931,7 +38931,7 @@ impl Interpreter {
                 let mut out = Vec::new();
                 if !self.seq_drain(l, &mut out, globals)? {
                     loop {
-                        if let Some(r) = crate::seqiter::lazy_pure_next(l) {
+                        if let Some(r) = self.lazy_core_next(l) {
                             match r {
                                 Some(x) => out.push(x),
                                 None => break,
@@ -42013,15 +42013,30 @@ impl Interpreter {
     /// Runs at the interpreter level because the wrapped source can be
     /// any VM iterable (generator, user `__next__`, native iterator) —
     /// and, like CPython's C itertools, stepping adds no Python frame.
+    /// A `StopIteration` that a callback raises ends the iteration, as a C
+    /// iterator's `tp_iternext` leaving it set does for its consumer
+    /// (`for x in map(f, xs)` stops where `f` raises it).
     fn lazy_iter_next(
         &mut self,
         l: &Rc<crate::object::PyLazyIter>,
         globals: &Rc<RefCell<DictData>>,
+    ) -> Result<Option<Object>, RuntimeError> {
+        match self.lazy_iter_step(l, globals) {
+            Err(e) if crate::seqiter::is_stop_iteration(&e) => Ok(None),
+            r => r,
+        }
+    }
+
+    /// [`Self::lazy_iter_next`]'s step.
+    fn lazy_iter_step(
+        &mut self,
+        l: &Rc<crate::object::PyLazyIter>,
+        globals: &Rc<RefCell<DictData>>,
     ) -> Result<Option<Object>, RuntimeError> {
         use crate::object::LazyIterKind;
-        // Steps that run no code first (pure state machines, and native
-        // sources with an item ready), under one borrow.
-        if let Some(r) = crate::seqiter::lazy_pure_next(l) {
+        // Steps that run no code first (pure state machines, native
+        // sources with an item ready, pure callbacks), under one borrow.
+        if let Some(r) = self.lazy_core_next(l) {
             return Ok(r);
         }
         // The builtin `map`/`filter`/`zip`/`enumerate` adapters.
@@ -56486,11 +56501,24 @@ impl Interpreter {
     /// call to the full machinery, nothing having happened.
     #[inline]
     fn call_pure_leaf(&self, f: &PyFunction, args: &[Object]) -> Option<Object> {
+        self.call_pure_leaf_with(f, None, args)
+    }
+
+    /// [`Self::call_pure_leaf`] with `recv`, when given, as the first
+    /// argument (a bound method's receiver).
+    #[inline]
+    pub(crate) fn call_pure_leaf_with(
+        &self,
+        f: &PyFunction,
+        recv: Option<&Object>,
+        args: &[Object],
+    ) -> Option<Object> {
         const MAX_ARGS: usize = 6;
         // SAFETY: a function's code is only replaced under the GIL, and
         // nothing here runs code that could replace it.
         let code = unsafe { &*f.code.as_ptr() };
-        if args.len() > MAX_ARGS
+        let n = args.len() + usize::from(recv.is_some());
+        if n > MAX_ARGS
             || !code_is_pure_leaf(code)
             || !pure_leaf_warm(code)
             || crate::trace::any_observers_active()
@@ -56500,10 +56528,10 @@ impl Interpreter {
             return None;
         }
         let mut ptrs = [std::ptr::null::<Object>(); MAX_ARGS];
-        for (p, a) in ptrs.iter_mut().zip(args) {
+        for (p, a) in ptrs.iter_mut().zip(recv.into_iter().chain(args)) {
             *p = a;
         }
-        self.pure_leaf_eval::<false, false>(code, f, &ptrs[..args.len()])
+        self.pure_leaf_eval::<false, false>(code, f, &ptrs[..n])
     }
 
     /// The full argument binder for [`Self::call_python_owned`]: CPython's
diff --git a/crates/weavepy-vm/src/seqiter.rs b/crates/weavepy-vm/src/seqiter.rs
index cb84c2fa..45c32eac 100644
--- a/crates/weavepy-vm/src/seqiter.rs
+++ b/crates/weavepy-vm/src/seqiter.rs
@@ -96,6 +96,12 @@ fn pure_iter(obj: &Object) -> Option<Object> {
     }
 }
 
+/// Whether `e` is a `StopIteration` (or a subclass's) being raised.
+pub(crate) fn is_stop_iteration(e: &RuntimeError) -> bool {
+    matches!(e, RuntimeError::PyException(exc) if matches!(&exc.instance,
+        Object::Instance(i) if i.cls_raw().is_subclass_of(&builtin_types().stop_iteration)))
+}
+
 /// `"argument 1"` / `"arguments 1-N"`: the strict-mode mismatch wording.
 fn arg_range(count: usize) -> String {
     if count == 1 {
@@ -120,8 +126,9 @@ impl Interpreter {
         self.iter_next(it, globals)
     }
 
-    /// `func(v)` for an adapter's callback: a plain function through
-    /// the pure-leaf evaluator when it qualifies, else a full call.
+    /// `func(v)` for an adapter's callback: a pure callback in place (see
+    /// [`Self::pure_callback`]), a plain function or a method over one
+    /// bound directly, else a full call.
     #[inline]
     pub(crate) fn seq_call1(
         &mut self,
@@ -129,12 +136,18 @@ impl Interpreter {
         v: Object,
         globals: &Rc<RefCell<DictData>>,
     ) -> Result<Object, RuntimeError> {
-        if let Object::Function(f) = func {
-            if let Some(r) = self.call_pure_leaf(f, std::slice::from_ref(&v)) {
-                self.release(v);
-                return Ok(r);
+        if let Some(r) = self.pure_callback(func, std::slice::from_ref(&v)) {
+            self.release(v);
+            return Ok(r);
+        }
+        match func {
+            Object::Function(f) => return self.call_python_owned(f, vec![v], Vec::new()),
+            Object::BoundMethod(bm) if !bm.redispatch_descriptor => {
+                if let Object::Function(f) = &bm.function {
+                    return self.call_python_owned(f, vec![bm.receiver.clone(), v], Vec::new());
+                }
             }
-            return self.call_python_owned(f, vec![v], Vec::new());
+            _ => {}
         }
         let r = self.call(func, std::slice::from_ref(&v), &[], globals);
         // A dead item's allocation is recycled (a tuple's, as CPython's
@@ -169,15 +182,23 @@ impl Interpreter {
                 out.reserve(if filter { n.min(64) } else { n });
             }
         }
+        // (A callback's `StopIteration` ends the drain, as it ends a step:
+        // see `Interpreter::lazy_iter_next`.)
+        let stop = |e: RuntimeError| if is_stop_iteration(&e) { Ok(true) } else { Err(e) };
         if filter {
-            while let Some(v) = self.filter_next(&func, &source, globals)? {
-                out.push(v);
+            loop {
+                match self.filter_next(&func, &source, globals) {
+                    Ok(Some(v)) => out.push(v),
+                    Ok(None) => return Ok(true),
+                    Err(e) => return stop(e),
+                }
             }
-            return Ok(true);
         }
         while let Some(v) = self.seq_step_source(&source, globals)? {
-            let r = self.seq_call1(&func, v, globals)?;
-            out.push(r);
+            match self.seq_call1(&func, v, globals) {
+                Ok(r) => out.push(r),
+                Err(e) => return stop(e),
+            }
         }
         Ok(true)
     }
@@ -1765,60 +1786,231 @@ fn scalar_truth(v: &Object) -> Option<bool> {
     })
 }
 
+/// `a == b` for two `groupby` keys when it is settled natively (identity,
+/// two machine ints, strings or floats, two classes that compare by
+/// identity): `None` when it could need Python's `__eq__`.
+fn native_key_eq(a: &Object, b: &Object) -> Option<bool> {
+    if a.is_same(b) {
+        return Some(true);
+    }
+    Some(match (a, b) {
+        (Object::Int(x), Object::Int(y)) => x == y,
+        (Object::Str(x), Object::Str(y)) => crate::dictmap::key_bytes_eq(x.as_bytes(), y.as_bytes()),
+        (Object::Float(x), Object::Float(y)) => x == y,
+        (Object::Bool(x), Object::Bool(y)) => x == y,
+        (Object::Type(x), Object::Type(y))
+            if crate::object::type_hash_is_identity(x) && crate::object::type_hash_is_identity(y) =>
+        {
+            Rc::ptr_eq(x, y)
+        }
+        _ => return None,
+    })
+}
+
+/// The next item of a plain sequence cursor, without taking it (see
+/// [`PyIterator::pure_peek`]).
+#[inline]
+fn cursor_peek(c: &Rc<RefCell<PyIterator>>) -> Option<Object> {
+    // SAFETY: a read that runs no code; the view ends with this call.
+    unsafe { c.peek() }?.pure_peek()
+}
+
+/// Take the item [`cursor_peek`] found.
+#[inline]
+fn cursor_advance(c: &Rc<RefCell<PyIterator>>) -> Option<()> {
+    // SAFETY: as above; the item was found ready.
+    unsafe { c.peek_mut() }?.next_value().map(drop)
+}
+
 impl Interpreter {
-    /// [`lazy_pure_next`], and also a `map` or `filter` over a plain
-    /// sequence cursor whose callback is a warm pure-leaf function
-    /// (evaluated in place, as the core loop calls one): the item is
-    /// taken only once the callback has succeeded. `None` (nothing
-    /// consumed) for every other step.
-    pub(crate) fn lazy_core_next(&self, l: &PyLazyIter) -> Option<Option<Object>> {
-        if let Some(r) = lazy_pure_next(l) {
-            return Some(r);
+    /// `func(*args)` for an adapter's callback when the call runs no Python
+    /// code: a warm pure-leaf function or a bound method over one (evaluated
+    /// in place, as the core loop calls one), a registered leaf builtin or a
+    /// leaf builtin method bound to its receiver, or a builtin type whose
+    /// native constructor builds the value (`int`, `str`, `type`, ...).
+    /// `None` leaves the call to the full machinery, nothing having
+    /// happened (a pure call that would raise declines too, so the full
+    /// call raises it).
+    pub(crate) fn pure_callback(&self, func: &Object, args: &[Object]) -> Option<Object> {
+        match func {
+            Object::Function(f) => self.call_pure_leaf(f, args),
+            // (As the core loop's leaf builtin lane admits them: a function,
+            // or a method bound to its exact builtin receiver.)
+            Object::Builtin(b) if !b.binds_instance => {
+                let kind = self.leaf_call_kind(b)?;
+                self.leaf_builtin_call(kind, b, args)?.ok()
+            }
+            Object::Type(t) => self.leaf_type_call(t, args),
+            Object::BoundMethod(bm) if !bm.redispatch_descriptor => match &bm.function {
+                Object::Function(f) => self.call_pure_leaf_with(f, Some(&bm.receiver), args),
+                Object::Builtin(b)
+                    if b.binds_instance
+                        && crate::builtins::method_memo_tag(&bm.receiver).is_some() =>
+                {
+                    let kind = self.leaf_call_kind(b)?;
+                    let call = match args {
+                        [a] => [bm.receiver.clone(), a.clone()],
+                        _ => return None,
+                    };
+                    self.leaf_builtin_call(kind, b, &call)?.ok()
+                }
+                _ => None,
+            },
+            _ => None,
         }
-        // SAFETY: nothing below runs code that could reach the adapter
-        // (pure leaves call nothing); the view ends with this call.
-        let st = unsafe { l.state.peek() }?;
-        let peek = |c: &Rc<RefCell<PyIterator>>| -> Option<Object> {
-            // SAFETY: as above (a distinct cell).
-            unsafe { c.peek() }?.pure_peek()
+    }
+
+    /// `groupby_step` when taking the next value and computing its key
+    /// run no code: the value and its key, the value taken. `None` (nothing
+    /// consumed) otherwise.
+    fn groupby_step_pure(&self, source: &Object, keyfunc: &Object) -> Option<(Object, Object)> {
+        let Object::Iter(c) = source else {
+            return None;
         };
-        let advance = |c: &Rc<RefCell<PyIterator>>| -> Option<()> {
-            // SAFETY: as above; the item was found ready.
-            unsafe { c.peek_mut() }?.next_value().map(drop)
+        let value = cursor_peek(c)?;
+        let key = match keyfunc {
+            Object::None => value.clone(),
+            f => self.pure_callback(f, std::slice::from_ref(&value))?,
         };
+        cursor_advance(c)?;
+        Some((value, key))
+    }
+
+    /// One step of a native adapter that runs no Python code, for every
+    /// `FOR_ITER` (the core loop's, the frame JIT's, tier 2's) and ahead of
+    /// the interpreter's general step: [`lazy_pure_next`]'s state machines,
+    /// and a `map` or `filter` over plain sequence cursors whose callback
+    /// is pure (see [`Self::pure_callback`]; the item is taken only once
+    /// the callback has succeeded), an `enumerate` over such an adapter,
+    /// and a `groupby` or grouper whose key function and key comparisons
+    /// are native. `None` (nothing consumed, or only whole steps the
+    /// general path would have taken the same way) for every other step.
+    pub(crate) fn lazy_core_next(&self, l: &Rc<PyLazyIter>) -> Option<Option<Object>> {
+        // SAFETY: nothing below runs code that could reach the adapter
+        // (pure callbacks call nothing); the view ends with this call.
+        let st = unsafe { l.state.peek_mut() }?;
         match st {
-            LazyIterKind::Map {
-                func: Object::Function(f),
-                iters,
-                ..
-            } => {
-                let [Object::Iter(c)] = iters.as_slice() else {
-                    return None;
-                };
-                let v = peek(c)?;
-                let r = self.call_pure_leaf(f, std::slice::from_ref(&v))?;
-                advance(c)?;
-                Some(Some(r))
-            }
+            LazyIterKind::Map { func, iters, .. } => match iters.as_slice() {
+                [Object::Iter(c)] => {
+                    let v = cursor_peek(c)?;
+                    let r = self.pure_callback(func, std::slice::from_ref(&v))?;
+                    cursor_advance(c)?;
+                    Some(Some(r))
+                }
+                [Object::Iter(a), Object::Iter(b)] if !Rc::ptr_eq(a, b) => {
+                    let args = [cursor_peek(a)?, cursor_peek(b)?];
+                    let r = self.pure_callback(func, &args)?;
+                    cursor_advance(a)?;
+                    cursor_advance(b)?;
+                    Some(Some(r))
+                }
+                _ => None,
+            },
             LazyIterKind::Filter {
                 func,
                 source: Object::Iter(c),
             } => loop {
-                let v = peek(c)?;
-                let keep = match func {
+                let v = cursor_peek(c)?;
+                let keep = match &*func {
                     Object::None => scalar_truth(&v)?,
                     Object::Type(t) if Rc::ptr_eq(t, &builtin_types().bool_) => scalar_truth(&v)?,
-                    Object::Function(f) => {
-                        scalar_truth(&self.call_pure_leaf(f, std::slice::from_ref(&v))?)?
-                    }
-                    _ => return None,
+                    f => scalar_truth(&self.pure_callback(f, std::slice::from_ref(&v))?)?,
                 };
-                advance(c)?;
+                cursor_advance(c)?;
                 if keep {
                     return Some(Some(v));
                 }
             },
-            _ => None,
+            // `enumerate(map(...))` and the like: the source's own pure step.
+            LazyIterKind::Enumerate {
+                source: Object::LazyIter(src),
+                count,
+                count_big: None,
+            } if *count < i64::MAX => {
+                let Some(v) = self.lazy_core_next(src)? else {
+                    return Some(None);
+                };
+                let i = *count;
+                *count += 1;
+                Some(Some(Object::new_tuple_array([Object::Int(i), v])))
+            }
+            LazyIterKind::GroupBy {
+                source,
+                keyfunc,
+                tgtkey,
+                currkey,
+                currvalue,
+                currgrouper,
+                grouper_cls,
+            } => {
+                // `groupby_next`: skip to the next group, a whole step at a
+                // time (a decline midway leaves the state a general step
+                // would have left).
+                *currgrouper = None;
+                loop {
+                    match (&*currkey, &*tgtkey) {
+                        (None, _) => {}
+                        (Some(_), None) => break,
+                        (Some(curr), Some(tgt)) => {
+                            if !native_key_eq(tgt, curr)? {
+                                break;
+                            }
+                        }
+                    }
+                    let (value, key) = self.groupby_step_pure(source, keyfunc)?;
+                    let old = (currkey.replace(key), currvalue.replace(value));
+                    drop(old);
+                }
+                let key = currkey.clone()?;
+                let old = tgtkey.replace(key.clone());
+                let grouper = Rc::new(PyLazyIter {
+                    state: RefCell::new(LazyIterKind::Grouper {
+                        parent: l.clone(),
+                        tgtkey: key.clone(),
+                    }),
+                    cls: grouper_cls.clone(),
+                });
+                *currgrouper = Some(Rc::downgrade(&grouper));
+                drop(old);
+                Some(Some(Object::new_tuple_array([key, Object::LazyIter(grouper)])))
+            }
+            LazyIterKind::Grouper { parent, tgtkey } => {
+                // `_grouper_next`.
+                // SAFETY: as above (the parent's distinct cell).
+                let LazyIterKind::GroupBy {
+                    source,
+                    keyfunc,
+                    currkey,
+                    currvalue,
+                    currgrouper,
+                    ..
+                } = (unsafe { parent.state.peek_mut() })?
+                else {
+                    return Some(None);
+                };
+                if !currgrouper
+                    .as_ref()
+                    .is_some_and(|w| std::ptr::eq(w.as_ptr(), Rc::as_ptr(l)))
+                {
+                    return Some(None);
+                }
+                if currvalue.is_none() {
+                    let (value, key) = self.groupby_step_pure(source, keyfunc)?;
+                    let old = (currkey.replace(key), currvalue.replace(value));
+                    drop(old);
+                }
+                let Some(curr) = currkey.as_ref() else {
+                    return Some(None);
+                };
+                if !native_key_eq(tgtkey, curr)? {
+                    return Some(None);
+                }
+                let value = currvalue.take()?;
+                let old = currkey.take();
+                drop(old);
+                Some(Some(value))
+            }
+            _ => lazy_pure_next(l),
         }
     }
 }
diff --git a/tests/regrtest/test_adapter_for_iter.py b/tests/regrtest/test_adapter_for_iter.py
new file mode 100644
index 00000000..e2677706
--- /dev/null
+++ b/tests/regrtest/test_adapter_for_iter.py
@@ -0,0 +1,196 @@
+"""`for` loops over builtin adapters (`map`, `filter`, `enumerate`, `zip`,
+`itertools.groupby`, ...) stepped natively: results, exhaustion, errors
+and tracebacks from the callback, `StopIteration` raised by a callback,
+mutation of the source, and finalizer timing. Each loop runs enough times
+to warm the interpreter's fast paths and the compiled loop."""
+import gc
+import itertools
+import sys
+import traceback
+
+ROUNDS = 300
+
+
+def leaf(x):
+    return x + 1
+
+
+def nonleaf(x):
+    return abs(x) + len(str(x))
+
+
+class C:
+    def m(self, x):
+        return x * 2
+
+
+def check(name, make, want):
+    for _ in range(ROUNDS):
+        got = []
+        for x in make():
+            got.append(x)
+        assert got == want, (name, got[:10], want[:10])
+
+
+xs = list(range(40))
+ws = ["a", "bb", "", "ccc"] * 5
+check("map leaf", lambda: map(leaf, xs), [x + 1 for x in xs])
+check("map nonleaf", lambda: map(nonleaf, xs), [abs(x) + len(str(x)) for x in xs])
+check("map int", lambda: map(int, ws[:0] + ["1", "22", "-3"]), [1, 22, -3])
+check("map str", lambda: map(str, xs), [str(x) for x in xs])
+check("map len", lambda: map(len, ws), [len(w) for w in ws])
+check("map type", lambda: map(type, [1, "a", 2.0, None]), [int, str, float, type(None)])
+check("map bound", lambda: map(C().m, xs), [x * 2 for x in xs])
+check("map bound builtin", lambda: map({1: "x"}.get, [1, 2]), ["x", None])
+check("map two", lambda: map(lambda a, b: a * b, xs, xs[1:]), [a * b for a, b in zip(xs, xs[1:])])
+check("map same source", lambda: (lambda it: map(lambda a, b: (a, b), it, it))(iter(xs)),
+      [(xs[i], xs[i + 1]) for i in range(0, 40, 2)])
+check("filter None", lambda: filter(None, [0, 1, "", "a", None, 2.0]), [1, "a", 2.0])
+check("filter leaf", lambda: filter(lambda v: v % 3, xs), [x for x in xs if x % 3])
+check("filter nonleaf", lambda: filter(nonleaf, [0, -1, 2]), [0, -1, 2])
+check("filter len", lambda: filter(len, ws), [w for w in ws if w])
+check("enumerate map", lambda: enumerate(map(leaf, xs), 5), list(enumerate([x + 1 for x in xs], 5)))
+check("enumerate gen", lambda: enumerate(x for x in xs), list(enumerate(xs)))
+check("zip", lambda: zip(xs, ws), list(zip(xs, ws)))
+check("reversed", lambda: reversed(xs), xs[::-1])
+check("chain", lambda: itertools.chain(xs, ws), xs + ws)
+check("islice", lambda: itertools.islice(xs, 3, 30, 4), xs[3:30:4])
+keys = [1, 1, 2, 2, 2, 1, 3, 3]
+check("groupby", lambda: ((k, list(g)) for k, g in itertools.groupby(keys)),
+      [(1, [1, 1]), (2, [2, 2, 2]), (1, [1]), (3, [3, 3])])
+check("groupby type", lambda: ((k, len(list(g))) for k, g in itertools.groupby([1, 2, "a", 3.0, 4.0], type)),
+      [(int, 2), (str, 1), (float, 2)])
+check("groupby leaf key", lambda: ((k, list(g)) for k, g in itertools.groupby(xs[:9], lambda v: v // 3)),
+      [(0, [0, 1, 2]), (1, [3, 4, 5]), (2, [6, 7, 8])])
+# A grouper outlived by its groupby's advance is empty.
+for _ in range(ROUNDS):
+    groups = list(itertools.groupby(keys))
+    assert [list(g) for _, g in groups] == [[], [], [], []], [list(g) for _, g in groups]
+
+# A grouper's own `for`, stopping when the key changes.
+for _ in range(ROUNDS):
+    out = []
+    for k, g in itertools.groupby("aaabbc"):
+        n = 0
+        for _c in g:
+            n += 1
+        out.append((k, n))
+    assert out == [("a", 3), ("b", 2), ("c", 1)], out
+
+
+# A raise in the callback propagates with its traceback.
+def boom(x):
+    if x == 7:
+        raise ValueError("boom %d" % x)
+    return x
+
+
+for _ in range(ROUNDS):
+    seen = []
+    try:
+        for v in map(boom, xs):
+            seen.append(v)
+    except ValueError as e:
+        assert str(e) == "boom 7"
+        names = [f.name for f in traceback.extract_tb(e.__traceback__)]
+        assert names[-1] == "boom", names
+    else:
+        raise AssertionError("no raise")
+    assert seen == list(range(7))
+
+# A pure leaf that raises (division by zero) raises from the loop.
+for _ in range(ROUNDS):
+    try:
+        for v in map(lambda d: 10 // d, [5, 2, 0, 1]):
+            pass
+    except ZeroDivisionError:
+        pass
+    else:
+        raise AssertionError("no ZeroDivisionError")
+
+
+# StopIteration from a callback ends the loop (CPython's FOR_ITER clears it).
+def stopper(x):
+    if x == 5:
+        raise StopIteration
+    return x
+
+
+for _ in range(ROUNDS):
+    got = []
+    for v in map(stopper, xs):
+        got.append(v)
+    assert got == [0, 1, 2, 3, 4], got
+    got = []
+    for v in filter(stopper, xs):
+        got.append(v)
+    assert got == [1, 2, 3, 4], got
+    # ... and the map stays exhausted only as far as its source is.
+    m = map(stopper, iter([5, 6]))
+    assert list(m) == []
+    assert list(m) == [6]
+
+# An exhausted adapter stays exhausted; length hints.
+for _ in range(ROUNDS):
+    m = map(leaf, xs)
+    for _v in m:
+        pass
+    assert list(m) == []
+    it = iter(xs)
+    m = map(leaf, it)
+    next(m)
+    assert it.__length_hint__() == 39
+    assert next(m) == 2
+
+# Mutating the source list during the loop.
+for _ in range(ROUNDS):
+    src = [1, 2, 3]
+    got = []
+    for v in map(leaf, src):
+        got.append(v)
+        if len(src) < 6:
+            src.append(v * 10)
+    assert got == [2, 3, 4, 21, 31, 41], got
+    src = [1, 2, 3, 4]
+    got = []
+    for v in filter(None, src):
+        got.append(v)
+        if src:
+            del src[-1]
+    assert got == [1, 2], got
+
+# Finalizers of items the callback drops run promptly.
+log = []
+
+
+class Fin:
+    def __init__(self, i):
+        self.i = i
+
+    def __del__(self):
+        log.append(self.i)
+
+
+def take(f):
+    return f.i
+
+
+for _ in range(50):
+    log.clear()
+    for i in map(take, [Fin(j) for j in range(5)]):
+        pass
+    gc.collect()
+    assert sorted(log) == list(range(5)), log
+
+# A set changing size during iteration still raises.
+for _ in range(ROUNDS):
+    s = set(range(10))
+    try:
+        for v in map(leaf, s):
+            s.add(v + 100)
+    except RuntimeError as e:
+        assert "changed size" in str(e)
+    else:
+        raise AssertionError("set size change not detected")
+
+print("adapter for-iter ok")

From bac2d018edc60bbc20c8310487246638d962e691 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 13:46:42 -0700
Subject: [PATCH 107/173] perf: probe str and int membership of sets and dicts
 in line

`leaf_contains` (the core loop's, the quiet loop's and the frame JIT's
`CONTAINS_OP` fast path) tests a `str` or machine `int` against a set,
frozenset or dict first, with the table's probe in line
(`SetMap::contains_hot`, `Table::get_index_hot`): no separate hashing
check, probe construction call or out-of-line table lookup.

Measured (instructions per item, macOS; CPython in parentheses):
  if x in s (int)   304 -> 255   (332)
  if x in s (str)   469 -> 400   (322)
tomllib_loads -2.0%, suite geomean unchanged at 1.437 (measured in one
build with the next commit's keyword change).

(Measured on the previous base, 785d1aa, before this branch was
rebuilt on d0f31c2.)
---
 crates/weavepy-vm/src/dictmap.rs | 22 +++++++++++++++++-----
 crates/weavepy-vm/src/lib.rs     | 26 ++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index d78ebf60..31b8b0d7 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -468,18 +468,23 @@ impl<V> Table<V> {
     /// fast paths, where the call would cost a fair share of the probe.
     #[inline(always)]
     pub fn get_hot<Q: Probe + ?Sized>(&self, key: &Q) -> Option<&V> {
+        let i = self.get_index_hot(key)?;
+        // SAFETY: a found position is in bounds.
+        Some(unsafe { &self.entries.get_unchecked(i).value })
+    }
+
+    /// [`Self::get_index_of`], always in line (see [`Self::get_hot`]).
+    #[inline(always)]
+    pub fn get_index_hot<Q: Probe + ?Sized>(&self, key: &Q) -> Option<usize> {
         if self.slots.is_empty() {
             if self.entries.is_empty() {
                 return None;
             }
             if let Some(found) = self.scan_unhashed(key) {
-                // SAFETY: a found position is in bounds.
-                return found.map(|i| unsafe { &self.entries.get_unchecked(i).value });
+                return found;
             }
         }
-        let i = self.find_hashed(key.probe_hash(), key).ok()?;
-        // SAFETY: a found position is in bounds.
-        Some(unsafe { &self.entries.get_unchecked(i).value })
+        self.find_hashed(key.probe_hash(), key).ok()
     }
 
     #[inline]
@@ -1398,6 +1403,13 @@ impl SetMap {
         self.find(key).is_some()
     }
 
+    /// [`Self::contains`], always in line, for a probe whose hash runs no
+    /// code (the interpreter's membership fast paths).
+    #[inline(always)]
+    pub fn contains_hot<Q: Probe + ?Sized>(&self, key: &Q) -> bool {
+        self.0.get_index_hot(key).is_some()
+    }
+
     #[inline]
     pub fn get_index_of<Q: Probe + ?Sized>(&self, key: &Q) -> Option<usize> {
         self.find(key)
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 861a5665..d94aba93 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -25007,6 +25007,32 @@ impl Interpreter {
     /// hands the instruction to the full handler (which restarts the scan:
     /// nothing observable happened here).
     fn leaf_contains(container: &Object, item: &Object) -> Option<bool> {
+        // A `str` or machine `int` in a set or dict, the commonest test,
+        // probed in line.
+        if let (
+            Object::Set(_) | Object::FrozenSet(_) | Object::Dict(_),
+            Object::Str(_) | Object::Int(_),
+        ) = (container, item)
+        {
+            if let Some(probe) = crate::object::LeafProbe::new(item) {
+                let found = match container {
+                    // SAFETY: a read between two instructions (the probe
+                    // runs no code).
+                    Object::Set(st) => unsafe { st.peek() }.map(|m| m.contains_hot(&probe)),
+                    Object::FrozenSet(fs) => Some(fs.contains_hot(&probe)),
+                    // SAFETY: as above.
+                    Object::Dict(d) => {
+                        unsafe { d.peek() }.map(|m| m.get_index_hot(&probe).is_some())
+                    }
+                    _ => None,
+                };
+                if let Some(found) = found {
+                    if found || probe.miss_is_exact() {
+                        return Some(found);
+                    }
+                }
+            }
+        }
         let member = Self::leaf_member_eq;
         match container {
             Object::List(l) => {

From 748653085932dd6cf1e0c56ae9c47333b0f43f2f Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 13:46:56 -0700
Subject: [PATCH 108/173] perf: bind lane keyword calls by the call site's name
 objects

The core loop's call lane (`CALL_KW` and `CALL_FUNCTION_EX` of a Python
function or method the inline and direct paths decline, such as a
callee with cell variables) built a `String` per keyword and bound
through the general binder. A Python callee now binds from the keyword
names' own `str` objects (`call_python_kw_names`) by the same plan the
`CALL_KW` specialization caches: `specialize::kw_perm`, now shared by
the specializer and this lane (each keyword tried first at the
parameter its position gives it, then by name), filled in by
`kw_names_fill_locals`, the binder the inline and direct keyword calls
use; `**kwargs` collects the rest under the same key objects. Any call
that needs an error, `*args`, an unknown or repeated or
positional-only name, or replaced defaults still takes the general
binder, which raises exactly as before.

Measured on the old base (785d1aa), with a binder of its own that this
version replaces by the shared one: urllib.parse.parse_qs("q=17&r=x")
79.5k -> 73.7k instructions per call under callgrind (CPython 3.13
21.2k).
---
 crates/weavepy-vm/src/lib.rs            | 117 +++++++++++++++++++++---
 crates/weavepy-vm/src/specialize.rs     |  68 +++++++++-----
 tests/regrtest/test_kw_binding_names.py | 108 ++++++++++++++++++++++
 3 files changed, 260 insertions(+), 33 deletions(-)
 create mode 100644 tests/regrtest/test_kw_binding_names.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index d94aba93..0fb3a4ab 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -16242,7 +16242,10 @@ impl Interpreter {
         }
         let ops = &frame.stack[callee_at..];
         let mut args: Vec<Object> = Vec::new();
-        let mut kwargs: Vec<(String, Object)> = Vec::new();
+        // The keywords' names are the call's own `str` objects (the site's
+        // names tuple, the `**` mapping's keys).
+        let mut kwnames: Vec<Object> = Vec::new();
+        let mut kwvals: Vec<Object> = Vec::new();
         if !matches!(ops[1], Object::Unbound) {
             args.push(ops[1].clone());
         }
@@ -16254,10 +16257,11 @@ impl Interpreter {
             let npos = values.len() - names.len();
             args.extend(values[..npos].iter().cloned());
             for (name, v) in names.iter().zip(&values[npos..]) {
-                let Object::Str(name) = name else {
+                if !matches!(name, Object::Str(_)) {
                     return false;
-                };
-                kwargs.push((name.to_string(), v.clone()));
+                }
+                kwnames.push(name.clone());
+                kwvals.push(v.clone());
             }
         } else {
             let Object::Tuple(items) = &ops[2] else {
@@ -16271,16 +16275,17 @@ impl Interpreter {
                         return false;
                     };
                     for (k, v) in d.iter() {
-                        let Object::Str(name) = &k.0 else {
+                        if !matches!(k.0, Object::Str(_)) {
                             return false;
-                        };
-                        kwargs.push((name.to_string(), v.clone()));
+                        }
+                        kwnames.push(k.0.clone());
+                        kwvals.push(v.clone());
                     }
                 }
                 _ => return false,
             }
         }
-        if builtin && !kwargs.is_empty() {
+        if builtin && !kwnames.is_empty() {
             return false;
         }
         // Committed: the operands leave the stack.
@@ -16289,10 +16294,34 @@ impl Interpreter {
         let globals = frame.globals.clone();
         frame.pc = pc as u32 + 1;
         let pending = self.core_pending_enter(sw, frame, pc);
-        let result = self.call(&callee, &args, &kwargs, &globals);
+        // A Python function binds its keywords by name objects; anything
+        // else takes them as strings.
+        let target = match &callee {
+            Object::Function(f) if !kwnames.is_empty() => Some((f.clone(), None)),
+            Object::BoundMethod(bm) if !kwnames.is_empty() => match &bm.function {
+                Object::Function(f) => Some((f.clone(), Some(bm.receiver.clone()))),
+                _ => None,
+            },
+            _ => None,
+        };
+        let result = match target {
+            Some((f, receiver)) => {
+                if let Some(r) = receiver {
+                    args.insert(0, r);
+                }
+                self.call_python_kw_names(&f, args, &kwnames, kwvals)
+            }
+            None => {
+                let kwargs: Vec<(String, Object)> =
+                    kwnames.iter().map(Object::to_str).zip(kwvals).collect();
+                let r = self.call(&callee, &args, &kwargs, &globals);
+                drop(args);
+                drop(kwargs);
+                r
+            }
+        };
         self.core_pending_exit(sw, pending);
-        drop(args);
-        drop(kwargs);
+        drop(kwnames);
         drop(callee);
         for o in operands {
             self.release(o);
@@ -57007,6 +57036,72 @@ impl Interpreter {
                 },
             )?
         };
+        self.call_python_bound(f, code, positional)
+    }
+
+    /// A keyword call of `f`: the positional `args`, then `kwvals` named by
+    /// `names` (the call site's `str` objects, one per value). Bound by the
+    /// `CALL_KW` specialization's permutation ([`specialize::kw_perm`],
+    /// computed for this call: no `String` per keyword) through
+    /// [`Self::kw_names_fill_locals`] when the shape has one; anything else,
+    /// its errors included, through the full binder.
+    fn call_python_kw_names(
+        &mut self,
+        f: &Rc<PyFunction>,
+        args: Vec<Object>,
+        names: &[Object],
+        kwvals: Vec<Object>,
+    ) -> Result<Object, RuntimeError> {
+        let code = f.code();
+        let strs = || {
+            names.iter().map(|n| match n {
+                Object::Str(s) => &**s,
+                _ => "",
+            })
+        };
+        let plan = if self.pending_raw_kwargs.borrow().is_none()
+            && names.iter().all(|n| matches!(n, Object::Str(_)))
+        {
+            specialize::kw_perm(f, &code, args.len(), strs())
+        } else {
+            None
+        };
+        if let Some((perm, covered)) = plan {
+            let (eff_argc, kwc) = (args.len(), kwvals.len());
+            let mut stack: Vec<Object> = self.pooled_scratch();
+            stack.extend(args);
+            stack.extend(kwvals);
+            let mut positional: Vec<Object> = self.pooled_scratch();
+            Self::kw_names_fill_locals(
+                &mut positional,
+                &mut stack,
+                names,
+                f,
+                &code,
+                perm,
+                covered,
+                kwc,
+                eff_argc,
+            );
+            self.recycle_scratch(stack);
+            return self.call_python_bound(f, code, positional);
+        }
+        let kwargs = names
+            .iter()
+            .zip(kwvals)
+            .map(|(n, v)| (n.to_str(), v))
+            .collect();
+        self.call_python_owned(f, args, kwargs)
+    }
+
+    /// Run `f` (whose code is `code`) on its bound fast locals
+    /// `positional` (see [`Self::call_python_owned`]).
+    fn call_python_bound(
+        &mut self,
+        f: &Rc<PyFunction>,
+        code: Rc<CodeObject>,
+        positional: Vec<Object>,
+    ) -> Result<Object, RuntimeError> {
         // Keep native eligibility outside the argument binder so its layout
         // values need not stay live across keyword and default processing.
         #[cfg(feature = "jit")]
diff --git a/crates/weavepy-vm/src/specialize.rs b/crates/weavepy-vm/src/specialize.rs
index ee6ebbc6..825b74b3 100644
--- a/crates/weavepy-vm/src/specialize.rs
+++ b/crates/weavepy-vm/src/specialize.rs
@@ -523,15 +523,44 @@ pub fn attempt_specialize_call_kw(
     kw_names: &[(String, crate::object::Object)],
 ) -> InlineCache {
     let code = f.code();
-    // Positional parameters, then the keyword-only ones.
-    let npos = code.arg_count as usize;
-    let total = npos + code.kwonly_count as usize;
-    let kwc = kw_names.len();
     if code.is_generator
         || code.is_coroutine
         || code.is_async_generator
-        || code.has_varargs
         || !(code.cellvars.is_empty() && code.freevars.is_empty() && f.closure.is_empty())
+    {
+        return InlineCache::Cooldown(COOLDOWN);
+    }
+    match kw_perm(f, &code, argc, kw_names.iter().map(|(n, _)| n.as_str())) {
+        Some((perm, _)) => InlineCache::CallPyKwNames {
+            func_id: rc_id(f),
+            perm,
+            argc: argc as u8,
+            kwc: kw_names.len() as u8,
+        },
+        None => InlineCache::Cooldown(COOLDOWN),
+    }
+}
+
+/// The `CallPyKwNames` permutation of a keyword call of `f` (whose code is
+/// `code`) with `argc` positionals (a receiver included) and the keyword
+/// `names`, and the parameter slots the call fills: each keyword's slot in
+/// 4 bits (`KW_TO_VARKW` for one a `**kwargs` callee collects). Each name
+/// is tried first at the parameter its position gives it. `None` for any
+/// call the generic binder must take: `*args`, more than 8 keywords or 16
+/// parameter slots, replaced defaults, a keyword without a distinct
+/// keyword-bindable parameter (or `**kwargs`), a parameter left without a
+/// default.
+pub fn kw_perm<'a>(
+    f: &Rc<crate::object::PyFunction>,
+    code: &weavepy_compiler::CodeObject,
+    argc: usize,
+    names: impl ExactSizeIterator<Item = &'a str>,
+) -> Option<(u32, u32)> {
+    // Positional parameters, then the keyword-only ones.
+    let npos = code.arg_count as usize;
+    let total = npos + code.kwonly_count as usize;
+    let kwc = names.len();
+    if code.has_varargs
         || (f.defaults_maybe_overridden()
             && (f.slot("__defaults__").is_some() || f.slot("__kwdefaults__").is_some()))
         || total > 16
@@ -543,15 +572,18 @@ pub fn attempt_specialize_call_kw(
         || kwc > 8
         || argc > npos
     {
-        return InlineCache::Cooldown(COOLDOWN);
+        return None;
     }
     let mut perm: u32 = 0;
     let mut covered: u32 = (1u32 << argc) - 1;
-    for (j, (name, _)) in kw_names.iter().enumerate() {
-        let bindable = code.varnames[..total]
-            .iter()
-            .position(|v| v == name)
-            .filter(|&slot| slot >= code.posonly_count as usize);
+    for (j, name) in names.enumerate() {
+        let at = argc + j;
+        let bindable = if at < total && code.varnames[at] == name {
+            Some(at)
+        } else {
+            code.varnames[..total].iter().position(|v| v == name)
+        }
+        .filter(|&slot| slot >= code.posonly_count as usize);
         let slot = match bindable {
             Some(slot) => slot,
             // No keyword-bindable parameter of that name (a
@@ -561,23 +593,15 @@ pub fn attempt_specialize_call_kw(
                 perm |= KW_TO_VARKW << (4 * j);
                 continue;
             }
-            None => return InlineCache::Cooldown(COOLDOWN),
+            None => return None,
         };
         if covered & (1 << slot) != 0 {
-            return InlineCache::Cooldown(COOLDOWN);
+            return None;
         }
         perm |= (slot as u32) << (4 * j);
         covered |= 1 << slot;
     }
-    if !kw_uncovered_have_defaults(f, &code, covered) {
-        return InlineCache::Cooldown(COOLDOWN);
-    }
-    InlineCache::CallPyKwNames {
-        func_id: rc_id(f),
-        perm,
-        argc: argc as u8,
-        kwc: kwc as u8,
-    }
+    kw_uncovered_have_defaults(f, code, covered).then_some((perm, covered))
 }
 
 /// Every parameter slot `covered` leaves unbound carries a default:
diff --git a/tests/regrtest/test_kw_binding_names.py b/tests/regrtest/test_kw_binding_names.py
new file mode 100644
index 00000000..1cb92d96
--- /dev/null
+++ b/tests/regrtest/test_kw_binding_names.py
@@ -0,0 +1,108 @@
+"""Keyword calls bound from the call site's names: callees with closures
+(cells), bound methods, `**kwargs` and `*args`, `f(*a, **d)`, keywords out
+of parameter order, defaults, and every binding error, each run enough
+times to take the interpreter's warmed call paths."""
+
+ROUNDS = 200
+
+
+def make():
+    secret = 7
+
+    def cl(a, b=2, c=3, *, d=4, e=5):
+        def inner():
+            return secret + a
+        return (a, b, c, d, e, inner())
+
+    def cl_star(a, *rest, k=1, **kw):
+        def inner():
+            return secret
+        return (a, rest, k, sorted(kw.items()), inner())
+
+    def cl_posonly(a, b, /, c=0, **kw):
+        def inner():
+            return secret
+        return (a, b, c, sorted(kw.items()), inner())
+
+    return cl, cl_star, cl_posonly
+
+
+cl, cl_star, cl_posonly = make()
+
+
+class K:
+    def __init__(self):
+        self.tag = "k"
+
+    def m(self, x, y=10, *, z=20):
+        def inner():
+            return self.tag
+        return (x, y, z, inner())
+
+
+def expect_error(fn, msg):
+    try:
+        fn()
+    except TypeError as e:
+        assert msg in str(e), (msg, str(e))
+    else:
+        raise AssertionError("no TypeError: " + msg)
+
+
+k = K()
+for _ in range(ROUNDS):
+    assert cl(1, b=2) == (1, 2, 3, 4, 5, 8)
+    assert cl(1, c=30, b=20) == (1, 20, 30, 4, 5, 8)
+    assert cl(a=1) == (1, 2, 3, 4, 5, 8)
+    assert cl(1, e=50, d=40) == (1, 2, 3, 40, 50, 8)
+    assert cl(1, 2, 3, d=0) == (1, 2, 3, 0, 5, 8)
+    assert cl(*(1, 2), **{"e": 9}) == (1, 2, 3, 4, 9, 8)
+    assert cl_star(1, 2, 3, k=4, x=5, y=6) == (1, (2, 3), 4, [("x", 5), ("y", 6)], 7)
+    assert cl_star(1, x=5) == (1, (), 1, [("x", 5)], 7)
+    assert cl_star(a=1, k=2) == (1, (), 2, [], 7)
+    assert cl_star(*[1, 2], **{"q": 3}) == (1, (2,), 1, [("q", 3)], 7)
+    # A positional-only name given as a keyword lands in `**kw`.
+    assert cl_posonly(1, 2, a=3, c=4) == (1, 2, 4, [("a", 3)], 7)
+    assert k.m(1, z=3) == (1, 10, 3, "k")
+    assert k.m(x=1, y=2) == (1, 2, 20, "k")
+    m = k.m
+    assert m(1, y=5) == (1, 5, 20, "k")
+    # The `**kwargs` dictionary keeps the call's keys and order.
+    r = cl_star(0, b=1, a2=2, c=3)
+    assert r[3] == [("a2", 2), ("b", 1), ("c", 3)]
+
+    expect_error(lambda: cl(1, a=2), "got multiple values for argument 'a'")
+    expect_error(lambda: cl(1, f=2), "got an unexpected keyword argument 'f'")
+    expect_error(lambda: cl(b=2), "missing 1 required positional argument: 'a'")
+    expect_error(lambda: cl(1, 2, 3, 4, d=1), "takes from 1 to 3 positional arguments but 4 positional arguments (and 1 keyword-only argument) were given")
+    expect_error(lambda: cl_posonly(1, c=2), "missing 1 required positional argument: 'b'")
+    expect_error(lambda: k.m(1, x=2), "got multiple values for argument 'x'")
+    expect_error(lambda: k.m(y=2), "missing 1 required positional argument: 'x'")
+    expect_error(lambda: cl(**{"a": 1, "zz": 2}), "got an unexpected keyword argument 'zz'")
+
+
+def posonly_only(a, /, b):
+    def inner():
+        return a
+    return inner() + b
+
+
+for _ in range(ROUNDS):
+    assert posonly_only(1, b=2) == 3
+    expect_error(lambda: posonly_only(a=1, b=2), "positional-only arguments passed as keyword arguments: 'a'")
+
+
+# Replaced defaults are honored.
+def with_defaults(a, b=1, *, c=2):
+    def inner():
+        return a
+    return (inner(), b, c)
+
+
+for i in range(ROUNDS):
+    if i == ROUNDS // 2:
+        with_defaults.__defaults__ = (100,)
+        with_defaults.__kwdefaults__ = {"c": 200}
+    want = (5, 1, 2) if i < ROUNDS // 2 else (5, 100, 200)
+    assert with_defaults(a=5) == want, (i, with_defaults(a=5))
+print("kw binding names ok")

From a67eacdb69fc472fbc684f047190d71c140afba9 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 15:53:35 -0700
Subject: [PATCH 109/173] perf: cheaper pure adapter steps over sequence
 cursors

A pure `map`/`filter` step peeks its source's next item and, once the
callback succeeded, steps past it with `PyIterator::pure_skip` (an index
bump) instead of a second `next_value` that clones the item again and
can fail. A two-source `zip` over plain cursors takes both items in one
pass (`cursor_pair`), refilling its result tuple when it holds it alone,
for the core loop's fused pair and every other `FOR_ITER`. A `filter`
callback whose truth test may still decline runs only if it has no
effect beyond its result (no `list.append` or opaque leaf). A `filter`
or `enumerate` over a source only the interpreter can step declines at
once instead of trying the `itertools` state machines.

Measured under callgrind (instructions per item):
  for a, b in zip(xs, ys)      1427 -> 1127
  for p in zip(xs, ys)         1163 -> 865
  for x in map(int, xs)         973 -> 880
  for x in map(len, lists)     1163 -> 1057
  for x in filter(f, xs)       1426 -> 1339

(Measured on the previous base, 785d1aa, before this branch was
rebuilt on d0f31c2.)
---
 crates/weavepy-vm/src/frame_jit.rs |   3 +-
 crates/weavepy-vm/src/object.rs    |  23 +++++++
 crates/weavepy-vm/src/seqiter.rs   | 100 +++++++++++++++++++++++++----
 3 files changed, 113 insertions(+), 13 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 8d24576a..99ea831c 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1528,7 +1528,8 @@ unsafe extern "C" fn h_for_iter_rest(
         // A set's next element while the set keeps its size (a change,
         // and exhaustion, are the core loop's to report).
         PyIterator::Set { set, len, index } => {
-            let Ok(items) = set.try_borrow() else {
+            // SAFETY: as above (a distinct cell; the read runs no code).
+            let Some(items) = (unsafe { set.peek() }) else {
                 return 3;
             };
             if items.len() != *len {
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 06a165eb..5173e1d5 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -9636,6 +9636,29 @@ impl PyIterator {
         }
     }
 
+    /// Step past the item [`Self::pure_peek`] returned (as `next` steps a
+    /// sequence iterator, whatever the sequence holds now): `false` for
+    /// any other iterator, untouched.
+    #[inline]
+    pub fn pure_skip(&mut self) -> bool {
+        match self {
+            PyIterator::List {
+                index, owner: None, ..
+            }
+            | PyIterator::Tuple { index, .. } => *index += 1,
+            PyIterator::Range { current, step, .. } => *current = current.wrapping_add(*step),
+            PyIterator::Str { s, index } => match s.get(*index..).and_then(|r| r.chars().next()) {
+                Some(ch) => *index += ch.len_utf8(),
+                None => return false,
+            },
+            PyIterator::Reversed {
+                index, owner: None, ..
+            } if *index >= 0 => *index -= 1,
+            _ => return false,
+        }
+        true
+    }
+
     /// Whether this iterator can participate in a reference cycle and so
     /// should be enrolled with the cycle GC when created (CPython tracks
     /// these `*_iterator` objects). Scalar iterators (`range`, `str`,
diff --git a/crates/weavepy-vm/src/seqiter.rs b/crates/weavepy-vm/src/seqiter.rs
index 45c32eac..f6caada0 100644
--- a/crates/weavepy-vm/src/seqiter.rs
+++ b/crates/weavepy-vm/src/seqiter.rs
@@ -17,7 +17,7 @@ use crate::builtin_types::{builtin_types, BuiltinTypes};
 use crate::error::{type_error, value_error, RuntimeError};
 use crate::object::{BuiltinFn, DictData, DictKey, LazyIterKind, Object, PyIterator, PyLazyIter};
 use crate::types::{PyInstance, TypeObject};
-use crate::Interpreter;
+use crate::{Interpreter, LeafKind};
 
 /// Which builtin adapter type a class is (or derives from).
 #[derive(Clone, Copy, PartialEq, Eq)]
@@ -136,7 +136,7 @@ impl Interpreter {
         v: Object,
         globals: &Rc<RefCell<DictData>>,
     ) -> Result<Object, RuntimeError> {
-        if let Some(r) = self.pure_callback(func, std::slice::from_ref(&v)) {
+        if let Some(r) = self.pure_callback(func, std::slice::from_ref(&v), true) {
             self.release(v);
             return Ok(r);
         }
@@ -914,6 +914,26 @@ pub(crate) fn lazy_pure_next(l: &PyLazyIter) -> Option<Option<Object>> {
     else {
         return itertools_pure_next(l);
     };
+    // Two plain sequence cursors (`zip(xs, ys)`), the commonest shape.
+    if let [a, b] = iters.as_slice() {
+        if let Some((x, y)) = cursor_pair(a, b) {
+            if let Some(slots) = result
+                .as_mut()
+                .and_then(crate::shared_value::ThinArc::get_mut)
+                .filter(|slots| slots.len() == 2)
+            {
+                let old = (
+                    std::mem::replace(&mut slots[0], x),
+                    std::mem::replace(&mut slots[1], y),
+                );
+                drop(old);
+                return Some(Some(Object::Tuple(result.clone()?)));
+            }
+            let t = crate::object::TupleStorage::from_array([x, y]);
+            *result = Some(t.clone());
+            return Some(Some(Object::Tuple(t)));
+        }
+    }
     if !srcs_ready(iters) {
         return None;
     }
@@ -958,12 +978,36 @@ pub(crate) fn lazy_pure_pair(l: &PyLazyIter) -> Option<(Object, Object)> {
     let [a, b] = iters.as_slice() else {
         return None;
     };
+    if let Some(pair) = cursor_pair(a, b) {
+        return Some(pair);
+    }
     if !srcs_ready(iters) {
         return None;
     }
     Some((src_step(a)?, src_step(b)?))
 }
 
+/// The next items of two distinct plain sequence cursors, both taken; `None`
+/// (neither taken) unless both have one.
+#[inline]
+fn cursor_pair(a: &Object, b: &Object) -> Option<(Object, Object)> {
+    let (Object::Iter(ca), Object::Iter(cb)) = (a, b) else {
+        return None;
+    };
+    if Rc::ptr_eq(ca, cb) {
+        return None;
+    }
+    // SAFETY: two distinct cells; nothing below runs code, and the views
+    // end with this call.
+    let (ia, ib) = unsafe { (ca.peek_mut()?, cb.peek_mut()?) };
+    let x = ia.pure_peek()?;
+    let y = ib.pure_peek()?;
+    // (`pure_skip` steps past exactly what `pure_peek` found.)
+    ia.pure_skip();
+    ib.pure_skip();
+    Some((x, y))
+}
+
 /// The `itertools` state machines that touch no other object (`repeat`,
 /// `product`, `permutations`, `combinations`,
 /// `combinations_with_replacement`), stepped under one borrow: `None`
@@ -1815,11 +1859,11 @@ fn cursor_peek(c: &Rc<RefCell<PyIterator>>) -> Option<Object> {
     unsafe { c.peek() }?.pure_peek()
 }
 
-/// Take the item [`cursor_peek`] found.
+/// Step past the item [`cursor_peek`] found.
 #[inline]
 fn cursor_advance(c: &Rc<RefCell<PyIterator>>) -> Option<()> {
     // SAFETY: as above; the item was found ready.
-    unsafe { c.peek_mut() }?.next_value().map(drop)
+    unsafe { c.peek_mut() }?.pure_skip().then_some(())
 }
 
 impl Interpreter {
@@ -1830,14 +1874,40 @@ impl Interpreter {
     /// native constructor builds the value (`int`, `str`, `type`, ...).
     /// `None` leaves the call to the full machinery, nothing having
     /// happened (a pure call that would raise declines too, so the full
-    /// call raises it).
-    pub(crate) fn pure_callback(&self, func: &Object, args: &[Object]) -> Option<Object> {
+    /// call raises it). Without `effects`, only callbacks with no effect
+    /// beyond their result qualify (a caller that may yet decline after
+    /// the call: no `list.append`, `set.pop`, or opaque registered leaf).
+    pub(crate) fn pure_callback(
+        &self,
+        func: &Object,
+        args: &[Object],
+        effects: bool,
+    ) -> Option<Object> {
+        let admit = |kind: LeafKind| -> Option<LeafKind> {
+            let effect = matches!(
+                kind,
+                LeafKind::ListAppend
+                    | LeafKind::ListPop
+                    | LeafKind::ListExtend
+                    | LeafKind::ListInsert
+                    | LeafKind::ListReverse
+                    | LeafKind::ListRemove
+                    | LeafKind::SetAdd
+                    | LeafKind::SetDiscard
+                    | LeafKind::SetRemove
+                    | LeafKind::SetPop
+                    | LeafKind::Opaque
+                    | LeafKind::Scalar
+                    | LeafKind::Fast(_)
+            );
+            (effects || !effect).then_some(kind)
+        };
         match func {
             Object::Function(f) => self.call_pure_leaf(f, args),
             // (As the core loop's leaf builtin lane admits them: a function,
             // or a method bound to its exact builtin receiver.)
             Object::Builtin(b) if !b.binds_instance => {
-                let kind = self.leaf_call_kind(b)?;
+                let kind = admit(self.leaf_call_kind(b)?)?;
                 self.leaf_builtin_call(kind, b, args)?.ok()
             }
             Object::Type(t) => self.leaf_type_call(t, args),
@@ -1847,7 +1917,7 @@ impl Interpreter {
                     if b.binds_instance
                         && crate::builtins::method_memo_tag(&bm.receiver).is_some() =>
                 {
-                    let kind = self.leaf_call_kind(b)?;
+                    let kind = admit(self.leaf_call_kind(b)?)?;
                     let call = match args {
                         [a] => [bm.receiver.clone(), a.clone()],
                         _ => return None,
@@ -1870,7 +1940,7 @@ impl Interpreter {
         let value = cursor_peek(c)?;
         let key = match keyfunc {
             Object::None => value.clone(),
-            f => self.pure_callback(f, std::slice::from_ref(&value))?,
+            f => self.pure_callback(f, std::slice::from_ref(&value), true)?,
         };
         cursor_advance(c)?;
         Some((value, key))
@@ -1893,13 +1963,13 @@ impl Interpreter {
             LazyIterKind::Map { func, iters, .. } => match iters.as_slice() {
                 [Object::Iter(c)] => {
                     let v = cursor_peek(c)?;
-                    let r = self.pure_callback(func, std::slice::from_ref(&v))?;
+                    let r = self.pure_callback(func, std::slice::from_ref(&v), true)?;
                     cursor_advance(c)?;
                     Some(Some(r))
                 }
                 [Object::Iter(a), Object::Iter(b)] if !Rc::ptr_eq(a, b) => {
                     let args = [cursor_peek(a)?, cursor_peek(b)?];
-                    let r = self.pure_callback(func, &args)?;
+                    let r = self.pure_callback(func, &args, true)?;
                     cursor_advance(a)?;
                     cursor_advance(b)?;
                     Some(Some(r))
@@ -1914,7 +1984,10 @@ impl Interpreter {
                 let keep = match &*func {
                     Object::None => scalar_truth(&v)?,
                     Object::Type(t) if Rc::ptr_eq(t, &builtin_types().bool_) => scalar_truth(&v)?,
-                    f => scalar_truth(&self.pure_callback(f, std::slice::from_ref(&v))?)?,
+                    // (The truth test may yet decline: no effects.)
+                    f => {
+                        scalar_truth(&self.pure_callback(f, std::slice::from_ref(&v), false)?)?
+                    }
                 };
                 cursor_advance(c)?;
                 if keep {
@@ -2010,6 +2083,9 @@ impl Interpreter {
                 drop(old);
                 Some(Some(value))
             }
+            // A `filter` or `enumerate` over a source only the
+            // interpreter can step.
+            LazyIterKind::Filter { .. } | LazyIterKind::Enumerate { .. } => None,
             _ => lazy_pure_next(l),
         }
     }

From 263012b56f748e7aab749078191d888770282e50 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 15:53:56 -0700
Subject: [PATCH 110/173] perf: class membership probes in line

`cls in _atomic_types` (`copy.deepcopy`'s first test) took the general
leaf path: a hashing check, a probe whose class hash re-checked the
metaclass and went through `identity_hash`'s whole match, an
out-of-line table lookup and an out-of-line equality. The set and dict
fast path (`leaf_contains_table`, now inlined into the frame JIT's
`CONTAINS_OP` helper as well) takes a plain class too: one metaclass
check, its pointer hash (`pointer_hash`) and an in-line identity
compare.

Measured under callgrind (instructions per item):
  if type(o) in frozenset_of_types   1088 -> 913   (CPython 3.13: 403)
deepcopy -2.3% (macOS ins_suite; one build with the previous and next
commits).

(Measured on the previous base, 785d1aa, before this branch was
rebuilt on d0f31c2.)
---
 crates/weavepy-vm/src/frame_jit.rs |  4 +-
 crates/weavepy-vm/src/lib.rs       | 67 +++++++++++++++++++-----------
 crates/weavepy-vm/src/object.rs    | 40 ++++++++++++++----
 3 files changed, 76 insertions(+), 35 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 99ea831c..a6a879a3 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -2361,7 +2361,9 @@ unsafe fn contains_fast(item: *const Object, cont: *const Object, owned: u32) ->
         if owned != 0 && !(droppable(i) && droppable(c)) {
             return 2;
         }
-        let Some(found) = Interpreter::leaf_contains(c, i) else {
+        let Some(found) = Interpreter::leaf_contains_table(c, i)
+            .or_else(|| Interpreter::leaf_contains(c, i))
+        else {
             return 2;
         };
         if owned & 2 != 0 {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 0fb3a4ab..261f5065 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -25036,31 +25036,8 @@ impl Interpreter {
     /// hands the instruction to the full handler (which restarts the scan:
     /// nothing observable happened here).
     fn leaf_contains(container: &Object, item: &Object) -> Option<bool> {
-        // A `str` or machine `int` in a set or dict, the commonest test,
-        // probed in line.
-        if let (
-            Object::Set(_) | Object::FrozenSet(_) | Object::Dict(_),
-            Object::Str(_) | Object::Int(_),
-        ) = (container, item)
-        {
-            if let Some(probe) = crate::object::LeafProbe::new(item) {
-                let found = match container {
-                    // SAFETY: a read between two instructions (the probe
-                    // runs no code).
-                    Object::Set(st) => unsafe { st.peek() }.map(|m| m.contains_hot(&probe)),
-                    Object::FrozenSet(fs) => Some(fs.contains_hot(&probe)),
-                    // SAFETY: as above.
-                    Object::Dict(d) => {
-                        unsafe { d.peek() }.map(|m| m.get_index_hot(&probe).is_some())
-                    }
-                    _ => None,
-                };
-                if let Some(found) = found {
-                    if found || probe.miss_is_exact() {
-                        return Some(found);
-                    }
-                }
-            }
+        if let Some(found) = Self::leaf_contains_table(container, item) {
+            return Some(found);
         }
         let member = Self::leaf_member_eq;
         match container {
@@ -25126,6 +25103,46 @@ impl Interpreter {
         }
     }
 
+    /// [`Self::leaf_contains`]'s commonest tests, small enough to inline
+    /// into the frame JIT's helper: a `str`, machine `int` or plain class
+    /// in a set or dict (`cls in _atomic_types`), probed in line. `None`
+    /// for anything else, or a miss a key of another kind might explain.
+    #[inline(always)]
+    pub(crate) fn leaf_contains_table(container: &Object, item: &Object) -> Option<bool> {
+        // A `str`, machine `int` or plain class in a set or dict, the
+        // commonest tests (`cls in _atomic_types`), probed in line.
+        if let (
+            Object::Set(_) | Object::FrozenSet(_) | Object::Dict(_),
+            Object::Str(_) | Object::Int(_) | Object::Type(_),
+        ) = (container, item)
+        {
+            let probe = match item {
+                Object::Type(t) => crate::object::type_hash_is_identity(t)
+                    .then(|| crate::object::LeafProbe::identity_type(item)),
+                _ => crate::object::LeafProbe::new(item),
+            };
+            if let Some(probe) = probe {
+                let found = match container {
+                    // SAFETY: a read between two instructions (the probe
+                    // runs no code).
+                    Object::Set(st) => unsafe { st.peek() }.map(|m| m.contains_hot(&probe)),
+                    Object::FrozenSet(fs) => Some(fs.contains_hot(&probe)),
+                    // SAFETY: as above.
+                    Object::Dict(d) => {
+                        unsafe { d.peek() }.map(|m| m.get_index_hot(&probe).is_some())
+                    }
+                    _ => None,
+                };
+                if let Some(found) = found {
+                    if found || probe.miss_is_exact() {
+                        return Some(found);
+                    }
+                }
+            }
+        }
+        None
+    }
+
     /// Whether hashing `item` runs no Python code: a native scalar, a
     /// class whose metaclass keeps `type`'s `__hash__` and `__eq__`, or an
     /// instance hashed by identity. (Equality with a stored key that
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 5173e1d5..b95db8b1 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -3551,6 +3551,22 @@ impl<'a> LeafProbe<'a> {
         })
     }
 
+    /// A probe for a class that hashes and compares by identity (see
+    /// [`type_hash_is_identity`]), checked by the caller.
+    #[inline(always)]
+    pub(crate) fn identity_type(key: &'a Object) -> Self {
+        let hash = match key {
+            // (As `identity_hash` hashes a class.)
+            Object::Type(t) => pointer_hash(Rc::as_ptr(t).cast()),
+            _ => identity_hash(key),
+        };
+        Self {
+            key,
+            hash,
+            foreign: std::cell::Cell::new(false),
+        }
+    }
+
     /// [`Self::new`]'s hash for a key other than a `str` or an `int`.
     #[inline(never)]
     fn other_hash(key: &Object) -> Option<i64> {
@@ -3633,6 +3649,8 @@ impl crate::dictmap::Probe for LeafProbe<'_> {
                 SharedStr::ptr_eq(a, b) || crate::dictmap::key_bytes_eq(a.as_bytes(), b.as_bytes())
             }
             (Object::Int(a), Object::Int(b)) => a == b,
+            // The same class (a class probe meets its own entry).
+            (Object::Type(a), Object::Type(b)) if Rc::ptr_eq(a, b) => true,
             _ => self.other_eq(key),
         }
     }
@@ -13146,20 +13164,24 @@ pub fn py_wstr_hash(cps: &[u32]) -> i64 {
     py_hash_bytes_slice(&buf)
 }
 
+/// CPython's pointer hash of the allocation at `p` (see [`identity_hash`]).
+#[inline(always)]
+pub(crate) fn pointer_hash(p: *const ()) -> i64 {
+    let u = p as usize as u64;
+    let v = u.rotate_right(4) as i64;
+    if v == -1 {
+        -2
+    } else {
+        v
+    }
+}
+
 /// Identity-based hash for objects that hash by allocation identity in
 /// CPython (functions, types, modules, plain instances without a custom
 /// `__hash__`, …). Mirrors CPython's pointer hash: rotate so the low
 /// alignment zero-bits don't waste bucket entropy, remapping `-1` to `-2`.
 pub(crate) fn identity_hash(obj: &Object) -> i64 {
-    fn rot(p: *const ()) -> i64 {
-        let u = p as usize as u64;
-        let v = u.rotate_right(4) as i64;
-        if v == -1 {
-            -2
-        } else {
-            v
-        }
-    }
+    let rot = pointer_hash;
     match obj {
         Object::Function(r) => rot(Rc::as_ptr(r).cast()),
         Object::Builtin(r) => rot(Rc::as_ptr(r).cast()),

From 1c52c643e966902be3a20fa829724833521fda48 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 16:43:20 -0700
Subject: [PATCH 111/173] perf: dict iterators and unpacked pairs in the frame
 JIT's FOR_ITER

The frame JIT's `FOR_ITER` declined every dict and dict-view iterator,
so each step of `for k in d` or `for k, v in d.items()` left native code
for the core loop and came back. `h_for_iter_rest` now steps them as the
core loop's arm does (`core_dict_next`: a change of size, a reverse view
and a releasing exhaustion still go to the core loop).

A `FOR_ITER` followed by `UNPACK_SEQUENCE 2` is fused (as the core loop
fuses it): `h_for_iter_pair` writes the pair straight onto the stack and
the code continues past the unpacking, with no tuple built and none
taken apart, for a dict's items, `enumerate` over a cursor, a
two-source `zip`, and a list's or tuple's next item when it is an exact
pair. Any other step takes the unfused helpers.

Measured under callgrind, tier 2 off (instructions per item; CPython
3.13 in parentheses):
  for k, v in d.items(): t += v       686 -> 317   (412)
  for k in d: t += 1                  495 -> 302   (259)
  for v in d.values(): t += v         482 -> 304   (333)
  for k, v in d.items(): o.last = v   852 -> 446   (287)
  for i, x in enumerate(xs)           682 -> 390   (409)
  for a, b in zip(xs, ys)             668 -> 402
Suite geomean 1.435 -> 1.430 (deepcopy -3.2%, graph_search -1.8%,
oo_patterns -0.8%, raytrace -0.7%).

(Measured on the previous base, 785d1aa, before this branch was
rebuilt on d0f31c2.)
---
 crates/weavepy-vm/src/frame_jit.rs    | 118 ++++++++++++++++++++++++++
 tests/regrtest/test_for_iter_pairs.py | 116 +++++++++++++++++++++++++
 2 files changed, 234 insertions(+)
 create mode 100644 tests/regrtest/test_for_iter_pairs.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index a6a879a3..0ab13508 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1542,6 +1542,21 @@ unsafe extern "C" fn h_for_iter_rest(
             *index += 1;
             v
         }
+        // A dict or dict view's next key, value or `(key, value)` item, as
+        // the core loop's arm steps it (a change of size, a reverse view, an
+        // exhaustion that could release anything are its own).
+        view @ PyIterator::DictKeys { .. } => {
+            match Interpreter::core_dict_next(view, false, unique) {
+                crate::DictStep::Item(v, _) => v,
+                crate::DictStep::Exhausted => {
+                    // SAFETY: the iterator, held by the stack alone, leaves
+                    // it (its dict lives on).
+                    drop(unsafe { it.read() });
+                    return 2;
+                }
+                crate::DictStep::Decline => return 3,
+            }
+        }
         // Any other cursor whose step runs no code (a string's, bytes');
         // its exhaustion is the core loop's.
         other if other.pure_ready() => match other.next_value() {
@@ -1571,6 +1586,78 @@ unsafe extern "C" fn h_for_iter_rest(
     1
 }
 
+/// `FOR_ITER` fused with the `UNPACK_SEQUENCE 2` after it, for an iterator
+/// at `it` whose next step is a pair and runs no code: `enumerate` over a
+/// cursor, a dict's items, a two-source `zip`, or a list's or tuple's next
+/// item when it is an exact pair. `0` the pair's second item written to
+/// `out` and its first above it (as the unpacking leaves them, no tuple
+/// built), anything else declined untouched (the unfused helpers take it,
+/// exhaustion included).
+unsafe extern "C" fn h_for_iter_pair(it: *mut Object, out: *mut Object) -> u32 {
+    // SAFETY: the code passes an initialized slot with two free ones above
+    // it; nothing below runs code while the iterator is borrowed.
+    unsafe {
+        let pair = match &*it {
+            Object::Iter(rc) => {
+                let Some(iter) = rc.peek_mut() else {
+                    return 3;
+                };
+                match iter {
+                    PyIterator::List {
+                        items,
+                        index,
+                        owner: None,
+                    } => {
+                        let Some(Object::Tuple(t)) = items.peek().and_then(|xs| xs.get(*index))
+                        else {
+                            return 3;
+                        };
+                        let [a, b] = &t[..] else {
+                            return 3;
+                        };
+                        let pair = (crate::clone_hot(a), crate::clone_hot(b));
+                        *index += 1;
+                        pair
+                    }
+                    PyIterator::Tuple { items, index } => {
+                        let Some(Object::Tuple(t)) = items.get(*index) else {
+                            return 3;
+                        };
+                        let [a, b] = &t[..] else {
+                            return 3;
+                        };
+                        let pair = (crate::clone_hot(a), crate::clone_hot(b));
+                        *index += 1;
+                        pair
+                    }
+                    // (An items view's fused step gives the key apart; its
+                    // exhaustion, declined here, is the unfused helper's.)
+                    view @ PyIterator::DictKeys {
+                        kind: crate::object::DictViewKind::Items,
+                        ..
+                    } => match Interpreter::core_dict_next(view, true, false) {
+                        crate::DictStep::Item(v, Some(k)) => (k, v),
+                        crate::DictStep::Item(..) => unreachable!("an items view's fused step"),
+                        _ => return 3,
+                    },
+                    _ => match crate::seqiter::enumerate_pure_pair(&*it) {
+                        Some(pair) => pair,
+                        None => return 3,
+                    },
+                }
+            }
+            Object::LazyIter(l) => match crate::seqiter::lazy_pure_pair(l) {
+                Some(pair) => pair,
+                None => return 3,
+            },
+            _ => return 3,
+        };
+        out.write(pair.1);
+        out.add(1).write(pair.0);
+        0
+    }
+}
+
 /// The `FOR_ITER` at `pc` over the native adapter at `it` (the stack's top)
 /// whose step may run Python code (a `map` over a Python function), as the
 /// core loop's lane runs it (`Interpreter::core_for_iter_lane`): the frame
@@ -1618,6 +1705,15 @@ unsafe fn for_iter_lane(st: &mut State, it: *mut Object, pc: u64) -> u32 {
     }
 }
 
+/// Whether the `FOR_ITER` at `pc` of `code` unpacks a pair (the
+/// `UNPACK_SEQUENCE 2` after it), which [`h_for_iter_pair`] fuses.
+fn fused_pair_at(code: &CodeObject, pc: usize) -> bool {
+    let op = |pc: usize| code.instructions.get(pc).map(|i| (i.op, i.arg));
+    matches!(op(pc), Some((OpCode::ForIter, _)))
+        && matches!(op(pc + 1), Some((OpCode::UnpackSequence, 2)))
+        && pc + 2 < code.instructions.len()
+}
+
 /// [`h_for_iter`] switched to a generator's activation (`RELOAD`).
 const FOR_SWITCHED: u32 = 4;
 /// [`h_for_iter`]'s iterator raised (`RAISED`, the state past it).
@@ -7304,6 +7400,10 @@ impl<'a> Lower<'a> {
                 if t < n {
                     starts[t] = true;
                 }
+                if fused_pair_at(code, pc) {
+                    // Past the `UNPACK_SEQUENCE` a fused step skips.
+                    starts[pc + 2] = true;
+                }
                 if ins.op == OpCode::ForIter {
                     // The loop exit past `END_FOR` / `POP_ITER`.
                     let mut to = t;
@@ -9581,6 +9681,24 @@ impl<'a> Lower<'a> {
         let it_slot = self.depth - 1;
         let it = self.slot_addr(it_slot);
         let out_slot = self.slot_addr(self.depth);
+        // A loop that unpacks a pair (`for k, v in d.items()`): the pair
+        // straight onto the stack, past the `UNPACK_SEQUENCE`, when the
+        // step allows; any other step below.
+        if fused_pair_at(self.code, pc) {
+            let r = self
+                .call(h_for_iter_pair as *const () as usize, &[it, out_slot], true)
+                .expect("returns");
+            let pair_b = self.b.create_block();
+            let single = self.b.create_block();
+            self.b.ins().brif(r, single, &[], pair_b, &[]);
+            self.b.switch_to_block(pair_b);
+            let depth = self.depth;
+            self.depth += 2;
+            self.set_last(pc + 1);
+            self.goto(pc + 2);
+            self.depth = depth;
+            self.b.switch_to_block(single);
+        }
         let pcv = self.b.ins().iconst(types::I64, pc as i64);
         let r = self
             .call(
diff --git a/tests/regrtest/test_for_iter_pairs.py b/tests/regrtest/test_for_iter_pairs.py
new file mode 100644
index 00000000..98993e0e
--- /dev/null
+++ b/tests/regrtest/test_for_iter_pairs.py
@@ -0,0 +1,116 @@
+"""`for a, b in ...` loops in compiled code, whose `FOR_ITER` hands the pair
+straight to the stack (dict items, `enumerate`, `zip`, lists and tuples of
+pairs) and `for` loops over dicts and their views: values, unpacking errors,
+mutation during iteration, and exhaustion. Each loop runs enough times for
+the compiled loop to take it."""
+from collections import OrderedDict
+
+ROUNDS = 400
+
+
+def items_sum(d):
+    t = 0
+    for k, v in d.items():
+        t += v
+    return t
+
+
+def keys_list(d):
+    out = []
+    for k in d:
+        out.append(k)
+    return out
+
+
+def values_sum(d):
+    t = 0
+    for v in d.values():
+        t += v
+    return t
+
+
+def pairs_of(seq):
+    out = []
+    for a, b in seq:
+        out.append((b, a))
+    return out
+
+
+def enum_pairs(seq):
+    out = []
+    for i, x in enumerate(seq):
+        out.append(i * 10 + x)
+    return out
+
+
+def zip_pairs(xs, ys):
+    out = []
+    for a, b in zip(xs, ys):
+        out.append(a - b)
+    return out
+
+
+d = {"k%d" % i: i for i in range(30)}
+od = OrderedDict((str(i), i) for i in range(10))
+for _ in range(ROUNDS):
+    assert items_sum(d) == sum(range(30))
+    assert keys_list(d) == ["k%d" % i for i in range(30)]
+    assert values_sum(d) == sum(range(30))
+    assert items_sum(od) == 45
+    assert pairs_of([(1, 2), (3, 4)]) == [(2, 1), (4, 3)]
+    assert pairs_of(((1, "a"), (2, "b"))) == [("a", 1), ("b", 2)]
+    assert pairs_of(["ab", "cd"]) == [("b", "a"), ("d", "c")]
+    assert pairs_of([[1, 2], (3, 4)]) == [(2, 1), (4, 3)]
+    assert enum_pairs([5, 6, 7]) == [5, 16, 27]
+    assert zip_pairs([5, 6, 7], [1, 1]) == [4, 5]
+    assert items_sum({}) == 0
+
+    # A wrong-sized item raises from the unpacking.
+    for bad in ([(1, 2), (1, 2, 3)], [(1, 2), 5], ((1, 2), (3,))):
+        try:
+            pairs_of(bad)
+        except (ValueError, TypeError):
+            pass
+        else:
+            raise AssertionError("bad unpack not detected: %r" % (bad,))
+
+    # Changing a dict's size during iteration raises.
+    m = dict(d)
+    try:
+        for k, v in m.items():
+            if v == 5:
+                m["new"] = 1
+    except RuntimeError as e:
+        assert "changed size during iteration" in str(e), e
+    else:
+        raise AssertionError("dict size change not detected")
+    m = dict(d)
+    try:
+        for k in m:
+            if k == "k3":
+                del m["k4"]
+    except RuntimeError:
+        pass
+    else:
+        raise AssertionError("dict deletion not detected")
+    # Replacing values in place is allowed.
+    m = dict(d)
+    for k, v in m.items():
+        m[k] = v + 1
+    assert items_sum(m) == sum(range(30)) + 30
+
+    # A list mutated mid-loop: the iteration sees the change.
+    seq = [(1, 2), (3, 4)]
+    out = []
+    for a, b in seq:
+        out.append(a)
+        if len(seq) < 4:
+            seq.append((a + 10, b))
+    assert out == [1, 3, 11, 13], out
+
+    # An exhausted items iterator stays exhausted.
+    it = iter(d.items())
+    for k, v in it:
+        pass
+    assert list(it) == []
+print("for-iter pairs ok")

From 58a0a9c64b1ca848a67aeb61164440a6d4770464 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 17:34:16 -0700
Subject: [PATCH 112/173] perf: retire shared exhausted iterators in the frame
 JIT's FOR_ITER

A generator expression's loop shares its `.0` iterator with the
generator's local, so the frame JIT's `FOR_ITER` saw a non-unique
iterator at the loop's end and left the exhaustion to the core loop: one
exit from native code per generator-expression run. The core loop's arm
retires an exhausted range, string or bytes iterator however it's
shared (its release runs no code); the frame JIT's now does too, and
also retires an exhausted list iterator it detaches when the list holds
only plain scalars, and an exhausted set iterator whose release can't
free its set.

Measured: nqueens -3.9%, sudoku -2.5% (macOS ins_suite); every
`n_queens.<locals>.<genexpr>`, `permutations.<locals>.<genexpr>` and
sudoku `assign`/`eliminate` generator-expression `FOR_ITER` exit gone
(WEAVEPY_FRAME_JIT_STATS). Suite geomean 1.430 -> 1.430.

(Measured on the previous base, 785d1aa, before this branch was
rebuilt on d0f31c2.)
---
 crates/weavepy-vm/src/frame_jit.rs        | 50 ++++++++++++--------
 tests/regrtest/test_genexpr_exhaustion.py | 56 +++++++++++++++++++++++
 2 files changed, 87 insertions(+), 19 deletions(-)
 create mode 100644 tests/regrtest/test_genexpr_exhaustion.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 0ab13508..4f3fe911 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1419,9 +1419,10 @@ unsafe extern "C" fn h_for_iter_rest(
                 *current = current.wrapping_add(*step);
                 return 0;
             }
-            if !unique {
-                return 3;
-            }
+            // The exhausted iterator leaves the stack, as the core loop's
+            // arm retires it: it holds nothing, so its release (the last
+            // or, a generator expression's `.0` holding it too, not) runs
+            // no code.
             // SAFETY: the iterator leaves the stack.
             drop(unsafe { it.read() });
             return 2;
@@ -1444,8 +1445,13 @@ unsafe extern "C" fn h_for_iter_rest(
                 // A shared iterator (a generator expression's `.0`, which
                 // its local holds too) detaches from the list, as the core
                 // loop's arm does, so a later append can't resurrect it
-                // (when the list lives on: its release frees nothing).
-                None if owner.is_none() && (unique || crate::sync::Rc::strong_count(items) > 1) => {
+                // (when the list lives on, or holds only plain scalars: its
+                // release frees nothing that could finalize).
+                None if owner.is_none()
+                    && (unique
+                        || crate::sync::Rc::strong_count(items) > 1
+                        || xs.iter().all(plain_scalar)) =>
+                {
                     if !unique {
                         *items = crate::sync::Rc::new(crate::sync::RefCell::new(Vec::new()));
                     }
@@ -1466,16 +1472,7 @@ unsafe extern "C" fn h_for_iter_rest(
                 // tuple of names, `for k in key[:-1]`.)
                 if unique
                     && crate::shared_value::ThinArc::strong_count(items) == 1
-                    && !items.iter().all(|x| {
-                        matches!(
-                            x,
-                            Object::Int(_)
-                                | Object::Float(_)
-                                | Object::Bool(_)
-                                | Object::None
-                                | Object::Str(_)
-                        )
-                    })
+                    && !items.iter().all(plain_scalar)
                 {
                     return 3;
                 }
@@ -1536,7 +1533,14 @@ unsafe extern "C" fn h_for_iter_rest(
                 return 3;
             }
             let Some(k) = items.get_index(*index) else {
-                return 3;
+                // Exhausted: the iterator leaves the stack when its release
+                // can't free the set (shared, or the set lives on).
+                if unique && crate::sync::Rc::strong_count(set) == 1 {
+                    return 3;
+                }
+                // SAFETY: as above.
+                drop(unsafe { it.read() });
+                return 2;
             };
             let v = crate::clone_hot(&k.0);
             *index += 1;
@@ -1564,13 +1568,13 @@ unsafe extern "C" fn h_for_iter_rest(
             None => return 3,
         },
         // An exhausted string's or bytes' cursor leaves the stack, as a
-        // range's does: releasing it runs no code.
-        PyIterator::Str { s, index } if unique && *index >= s.len() => {
+        // range's does (shared or not): releasing it runs no code.
+        PyIterator::Str { s, index } if *index >= s.len() => {
             // SAFETY: as above.
             drop(unsafe { it.read() });
             return 2;
         }
-        PyIterator::Bytes { data, index } if unique && *index >= data.len() => {
+        PyIterator::Bytes { data, index } if *index >= data.len() => {
             // SAFETY: as above.
             drop(unsafe { it.read() });
             return 2;
@@ -1705,6 +1709,14 @@ unsafe fn for_iter_lane(st: &mut State, it: *mut Object, pc: u64) -> u32 {
     }
 }
 
+/// Whether releasing `x` frees nothing that could finalize.
+fn plain_scalar(x: &Object) -> bool {
+    matches!(
+        x,
+        Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None | Object::Str(_)
+    )
+}
+
 /// Whether the `FOR_ITER` at `pc` of `code` unpacks a pair (the
 /// `UNPACK_SEQUENCE 2` after it), which [`h_for_iter_pair`] fuses.
 fn fused_pair_at(code: &CodeObject, pc: usize) -> bool {
diff --git a/tests/regrtest/test_genexpr_exhaustion.py b/tests/regrtest/test_genexpr_exhaustion.py
new file mode 100644
index 00000000..20d44111
--- /dev/null
+++ b/tests/regrtest/test_genexpr_exhaustion.py
@@ -0,0 +1,56 @@
+"""Generator expressions run to exhaustion by compiled code, whose `.0`
+iterator the loop shares with the generator's local: ranges, strings,
+bytes, lists (of plain scalars, and of objects whose finalizers must run
+when the generator lets go of them), sets, and tuples. Each runs enough
+times for the compiled body to take it."""
+import gc
+
+ROUNDS = 500
+
+log = []
+
+
+class Fin:
+    def __init__(self, i):
+        self.i = i
+
+    def __del__(self):
+        log.append(self.i)
+
+
+def build(xs):
+    return tuple(x for x in xs)
+
+
+for r in range(ROUNDS):
+    n = r % 7
+    assert build(range(n)) == tuple(range(n))
+    assert build("abc"[: n % 4]) == tuple("abc"[: n % 4])
+    assert build(b"xyz"[: n % 4]) == tuple(b"xyz"[: n % 4])
+    assert build([1, 2.5, None, "s", True][:n]) == (1, 2.5, None, "s", True)[:n]
+    assert sorted(build({1, 2, 3, n})) == sorted({1, 2, 3, n})
+    assert build((n, n + 1)) == (n, n + 1)
+    assert set(c for c in "hello") == {"h", "e", "l", "o"}
+    assert sum(i for i in range(n)) == n * (n - 1) // 2
+    assert all(c.isalpha() for c in "abc")
+    assert not any(i < 0 for i in range(n))
+
+    # The objects of a temporary list a generator expression walked die
+    # as the finished generator lets go of the list's iterator.
+    log.clear()
+    g = (f.i for f in [Fin(1), Fin(2)])
+    assert list(g) == [1, 2]
+    assert sorted(log) == [1, 2], log
+    del g
+
+    # A set shrinking mid-walk still raises.
+    s = {1, 2, 3}
+    try:
+        for v in (x for x in s):
+            s.discard(3)
+            s.discard(2)
+    except RuntimeError:
+        pass
+    else:
+        raise AssertionError("set size change not detected")
+print("genexpr exhaustion ok")

From 0f77a880f3a71bb184c7885ae2b5ec5984e8d172 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 17:53:47 -0700
Subject: [PATCH 113/173] perf: step itertools sources over sequence cursors in
 one view

`ready_next`, the pure step of an `itertools` adapter's source
(`chain`'s active iterable, `islice`'s, `cycle`'s and `pairwise`'s),
takes a plain sequence cursor's next item with one view of it
(`pure_peek`, then `pure_skip`) instead of a readiness check and a
second `next_value`.

Measured under callgrind, tier 2 off (instructions per item; CPython
3.13 in parentheses):
  for a in chain(xs, ys)        1021 -> 825   (659)
  for a in islice(xs, 10, 90)    465 -> 389   (291)

(Measured on the previous base, 785d1aa, before this branch was
rebuilt on d0f31c2.)
---
 crates/weavepy-vm/src/seqiter.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/crates/weavepy-vm/src/seqiter.rs b/crates/weavepy-vm/src/seqiter.rs
index f6caada0..20a279ca 100644
--- a/crates/weavepy-vm/src/seqiter.rs
+++ b/crates/weavepy-vm/src/seqiter.rs
@@ -1177,6 +1177,16 @@ pub(crate) fn machine_step(st: &mut LazyIterKind) -> Option<Option<Object>> {
 /// The next item of a native iterator source when taking it runs no
 /// code and releases nothing (see [`PyIterator::pure_ready`]).
 fn ready_next(it: &Object) -> Option<Object> {
+    // A plain sequence cursor (`chain(xs, ys)`'s, `islice(xs, n)`'s), in
+    // one view.
+    if let Object::Iter(c) = it {
+        // SAFETY: a step that runs no code; the view ends with this call.
+        let cur = unsafe { c.peek_mut() }?;
+        if let Some(v) = cur.pure_peek() {
+            cur.pure_skip();
+            return Some(v);
+        }
+    }
     if !src_ready(it, false) {
         return None;
     }

From f206c1ea1c27c5676c6f2f061618413980e04770 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 18:46:51 -0700
Subject: [PATCH 114/173] perf: rustfmt the adapter, set and keyword changes

Formatting only, of the lines this branch's commits wrote (the files'
other unformatted lines are left as they were). No functional change.
---
 crates/weavepy-vm/src/dictmap.rs   | 63 +++++++++++++++++++++++-------
 crates/weavepy-vm/src/frame_jit.rs |  4 +-
 crates/weavepy-vm/src/seqiter.rs   | 24 ++++++++----
 3 files changed, 68 insertions(+), 23 deletions(-)

diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index 31b8b0d7..106b49bc 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -220,10 +220,7 @@ impl<V> Table<V> {
     /// the entries are laid down, then indexed once (no probe compares a
     /// key).
     #[inline]
-    pub fn from_unique_hashed(
-        n: usize,
-        mut entry: impl FnMut(usize) -> (i64, DictKey, V),
-    ) -> Self {
+    pub fn from_unique_hashed(n: usize, mut entry: impl FnMut(usize) -> (i64, DictKey, V)) -> Self {
         let mut entries: Vec<Bucket<V>> = Vec::with_capacity(n);
         let dst = entries.as_mut_ptr();
         for i in 0..n {
@@ -820,8 +817,15 @@ impl<V> Table<V> {
     }
 
     /// Remove the entries in `range`, yielding them.
-    pub fn drain<R: std::ops::RangeBounds<usize>>(&mut self, range: R) -> std::vec::IntoIter<(DictKey, V)> {
-        let out: Vec<(DictKey, V)> = self.entries.drain(range).map(|b| (b.key, b.value)).collect();
+    pub fn drain<R: std::ops::RangeBounds<usize>>(
+        &mut self,
+        range: R,
+    ) -> std::vec::IntoIter<(DictKey, V)> {
+        let out: Vec<(DictKey, V)> = self
+            .entries
+            .drain(range)
+            .map(|b| (b.key, b.value))
+            .collect();
         self.reindex();
         out.into_iter()
     }
@@ -884,7 +888,10 @@ impl<V> Table<V> {
     }
 
     /// Sort the entries with `cmp` (stable).
-    pub fn sort_by(&mut self, mut cmp: impl FnMut(&DictKey, &V, &DictKey, &V) -> std::cmp::Ordering) {
+    pub fn sort_by(
+        &mut self,
+        mut cmp: impl FnMut(&DictKey, &V, &DictKey, &V) -> std::cmp::Ordering,
+    ) {
         self.entries
             .sort_by(|a, b| cmp(&a.key, &a.value, &b.key, &b.value));
         self.reindex();
@@ -1132,12 +1139,33 @@ macro_rules! entry_iter {
     };
 }
 
-entry_iter!(Iter, std::slice::Iter<'a, Bucket<V>>, (&'a DictKey, &'a V), |b| (&b.key, &b.value));
-entry_iter!(IterMut, std::slice::IterMut<'a, Bucket<V>>, (&'a DictKey, &'a mut V), |b| (&b.key, &mut b.value));
-entry_iter!(IterMut2, std::slice::IterMut<'a, Bucket<V>>, (&'a mut DictKey, &'a mut V), |b| (&mut b.key, &mut b.value));
-entry_iter!(Keys, std::slice::Iter<'a, Bucket<V>>, &'a DictKey, |b| &b.key);
+entry_iter!(
+    Iter,
+    std::slice::Iter<'a, Bucket<V>>,
+    (&'a DictKey, &'a V),
+    |b| (&b.key, &b.value)
+);
+entry_iter!(
+    IterMut,
+    std::slice::IterMut<'a, Bucket<V>>,
+    (&'a DictKey, &'a mut V),
+    |b| (&b.key, &mut b.value)
+);
+entry_iter!(
+    IterMut2,
+    std::slice::IterMut<'a, Bucket<V>>,
+    (&'a mut DictKey, &'a mut V),
+    |b| (&mut b.key, &mut b.value)
+);
+entry_iter!(Keys, std::slice::Iter<'a, Bucket<V>>, &'a DictKey, |b| &b
+    .key);
 entry_iter!(Values, std::slice::Iter<'a, Bucket<V>>, &'a V, |b| &b.value);
-entry_iter!(ValuesMut, std::slice::IterMut<'a, Bucket<V>>, &'a mut V, |b| &mut b.value);
+entry_iter!(
+    ValuesMut,
+    std::slice::IterMut<'a, Bucket<V>>,
+    &'a mut V,
+    |b| &mut b.value
+);
 
 impl<V> Clone for Iter<'_, V> {
     fn clone(&self) -> Self {
@@ -1376,7 +1404,11 @@ impl SetMap {
     /// Add a probe's key if absent: the key is built (by `make`) only
     /// then. `true` when it was added.
     #[inline]
-    pub fn insert_probe<Q: Probe + ?Sized>(&mut self, key: &Q, make: impl FnOnce() -> DictKey) -> bool {
+    pub fn insert_probe<Q: Probe + ?Sized>(
+        &mut self,
+        key: &Q,
+        make: impl FnOnce() -> DictKey,
+    ) -> bool {
         match self.0.probe_entry(key) {
             ProbeEntry::Occupied(_) => false,
             ProbeEntry::Vacant(v) => {
@@ -1512,7 +1544,10 @@ impl SetMap {
     }
 
     /// Remove the keys in `range`, yielding them.
-    pub fn drain<R: std::ops::RangeBounds<usize>>(&mut self, range: R) -> impl Iterator<Item = DictKey> {
+    pub fn drain<R: std::ops::RangeBounds<usize>>(
+        &mut self,
+        range: R,
+    ) -> impl Iterator<Item = DictKey> {
         self.0.drain(range).map(|(k, ())| k)
     }
 
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 4f3fe911..e7ce963d 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -2469,8 +2469,8 @@ unsafe fn contains_fast(item: *const Object, cont: *const Object, owned: u32) ->
         if owned != 0 && !(droppable(i) && droppable(c)) {
             return 2;
         }
-        let Some(found) = Interpreter::leaf_contains_table(c, i)
-            .or_else(|| Interpreter::leaf_contains(c, i))
+        let Some(found) =
+            Interpreter::leaf_contains_table(c, i).or_else(|| Interpreter::leaf_contains(c, i))
         else {
             return 2;
         };
diff --git a/crates/weavepy-vm/src/seqiter.rs b/crates/weavepy-vm/src/seqiter.rs
index 20a279ca..2b4866bd 100644
--- a/crates/weavepy-vm/src/seqiter.rs
+++ b/crates/weavepy-vm/src/seqiter.rs
@@ -184,7 +184,13 @@ impl Interpreter {
         }
         // (A callback's `StopIteration` ends the drain, as it ends a step:
         // see `Interpreter::lazy_iter_next`.)
-        let stop = |e: RuntimeError| if is_stop_iteration(&e) { Ok(true) } else { Err(e) };
+        let stop = |e: RuntimeError| {
+            if is_stop_iteration(&e) {
+                Ok(true)
+            } else {
+                Err(e)
+            }
+        };
         if filter {
             loop {
                 match self.filter_next(&func, &source, globals) {
@@ -1849,11 +1855,14 @@ fn native_key_eq(a: &Object, b: &Object) -> Option<bool> {
     }
     Some(match (a, b) {
         (Object::Int(x), Object::Int(y)) => x == y,
-        (Object::Str(x), Object::Str(y)) => crate::dictmap::key_bytes_eq(x.as_bytes(), y.as_bytes()),
+        (Object::Str(x), Object::Str(y)) => {
+            crate::dictmap::key_bytes_eq(x.as_bytes(), y.as_bytes())
+        }
         (Object::Float(x), Object::Float(y)) => x == y,
         (Object::Bool(x), Object::Bool(y)) => x == y,
         (Object::Type(x), Object::Type(y))
-            if crate::object::type_hash_is_identity(x) && crate::object::type_hash_is_identity(y) =>
+            if crate::object::type_hash_is_identity(x)
+                && crate::object::type_hash_is_identity(y) =>
         {
             Rc::ptr_eq(x, y)
         }
@@ -1995,9 +2004,7 @@ impl Interpreter {
                     Object::None => scalar_truth(&v)?,
                     Object::Type(t) if Rc::ptr_eq(t, &builtin_types().bool_) => scalar_truth(&v)?,
                     // (The truth test may yet decline: no effects.)
-                    f => {
-                        scalar_truth(&self.pure_callback(f, std::slice::from_ref(&v), false)?)?
-                    }
+                    f => scalar_truth(&self.pure_callback(f, std::slice::from_ref(&v), false)?)?,
                 };
                 cursor_advance(c)?;
                 if keep {
@@ -2055,7 +2062,10 @@ impl Interpreter {
                 });
                 *currgrouper = Some(Rc::downgrade(&grouper));
                 drop(old);
-                Some(Some(Object::new_tuple_array([key, Object::LazyIter(grouper)])))
+                Some(Some(Object::new_tuple_array([
+                    key,
+                    Object::LazyIter(grouper),
+                ])))
             }
             LazyIterKind::Grouper { parent, tgtkey } => {
                 // `_grouper_next`.

From d3b59229d92a53e714daca03493ca230b1d2a7b1 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 18:39:08 -0700
Subject: [PATCH 115/173] perf: settle an lru_cache call on native keys under
 one cache hold

A bounded lru_cache wrapper called with an int or str argument (or a tuple
of them) now looks the key up, refreshes its recency, and counts the hit
or miss while it holds the cache and its recency links once, instead of
re-reading and re-validating the wrapper's fields for each step. A miss's
store evicts the least recent entry by putting the new key in its place
(DictMap::replace_index_unique) and moving that node to the recency tail,
instead of a swap-remove that renumbers the last entry followed by an
append. Other shapes (typed keys, foreign keys, transitions out of scalar
mode, exported recency storage) keep the existing path.

The wrapper's dictionary keys are now shared interned strings, so finding
a field at its known position is one pointer compare, and the recency
buffer's words are read with one bounds check each.

Measured (instructions per iteration, macOS, pinned hash seed):
- recursive collatz under lru_cache(256), mostly misses with evictions:
  41,279 -> 33,648 (CPython 3.14: 16,874)
- lru_cache hits on int keys: 2,140 -> 1,786 (CPython 3.14: 1,186)
---
 crates/weavepy-vm/src/dictmap.rs              |  49 ++++
 crates/weavepy-vm/src/stdlib/functools_mod.rs | 235 +++++++++++++++---
 .../src/stdlib/functools_mod/lru_order.rs     |  45 ++--
 3 files changed, 283 insertions(+), 46 deletions(-)

diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index 106b49bc..bc55c0ca 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -744,6 +744,31 @@ impl<V> Table<V> {
         Some((b.key, b.value))
     }
 
+    /// Put `key` (hash `hash`), known to be absent, in entry `i`'s place,
+    /// leaving every other entry where it is: the replaced key and value.
+    pub fn replace_index_unique(
+        &mut self,
+        i: usize,
+        hash: i64,
+        key: DictKey,
+        value: V,
+    ) -> Option<(DictKey, V)> {
+        let old = self.entries.get(i)?.hash;
+        if !self.slots.is_empty() {
+            let pos = self.slot_of(old, i);
+            self.erase_slot(pos);
+            let m = mix(hash);
+            let mask = self.slots.len() - 1;
+            let mut pos = (m >> self.shift) as usize;
+            while self.slots[pos] != 0 {
+                pos = (pos + 1) & mask;
+            }
+            self.slots[pos] = (m & !IX_MASK) | (i as u64 + 1);
+        }
+        let b = std::mem::replace(&mut self.entries[i], Bucket { hash, key, value });
+        Some((b.key, b.value))
+    }
+
     /// Remove the last entry.
     pub fn pop(&mut self) -> Option<(DictKey, V)> {
         let n = self.entries.len();
@@ -1733,4 +1758,28 @@ mod tests {
         assert_eq!(c.get(&k(5)).map(|o| matches!(o, Object::Int(5))), Some(true));
         assert!(c.get(&k(50)).is_none());
     }
+
+    #[test]
+    fn replace_in_place() {
+        for n in [5_i64, 40] {
+            let mut m = DictMap::new();
+            for i in 0..n {
+                m.insert(k(i * 1024), Object::Int(i));
+            }
+            for (step, i) in (0..n).rev().enumerate() {
+                let new = 100_000 + step as i64;
+                let key = k(new);
+                let hash = key.probe_hash();
+                let at = (i as usize * 7) % m.len();
+                let (old, _) = m
+                    .replace_index_unique(at, hash, key, Object::Int(new))
+                    .expect("in range");
+                assert!(m.get(&old).is_none());
+                assert_eq!(m.get_index_of(&k(new)), Some(at));
+                assert_eq!(m.len(), n as usize);
+                check(&m);
+            }
+            assert!(m.replace_index_unique(n as usize, 0, k(-1), Object::None).is_none());
+        }
+    }
 }
diff --git a/crates/weavepy-vm/src/stdlib/functools_mod.rs b/crates/weavepy-vm/src/stdlib/functools_mod.rs
index f393e5d3..de3ae590 100644
--- a/crates/weavepy-vm/src/stdlib/functools_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/functools_mod.rs
@@ -496,10 +496,30 @@ impl LruField {
         }
     }
 
-    /// Whether `key` names this field.
+    /// The interned name every wrapper's dictionary is built with.
+    #[inline]
+    fn key(self) -> &'static crate::shared_value::SharedStr {
+        static KEYS: std::sync::OnceLock<[crate::shared_value::SharedStr; 7]> =
+            std::sync::OnceLock::new();
+        &KEYS.get_or_init(|| {
+            LruField::ALL.map(|field| match crate::stdlib::sys::intern_name(field.name()) {
+                Object::Str(s) => s,
+                _ => unreachable!("interned names are strs"),
+            })
+        })[self as usize]
+    }
+
+    /// The field's key, for a new entry.
+    fn key_object(self) -> DictKey {
+        DictKey(Object::Str(self.key().clone()))
+    }
+
+    /// Whether `key` names this field: the interned name itself (one
+    /// compare), or an equal string.
     #[inline]
     fn is(self, key: &DictKey) -> bool {
-        matches!(&key.0, Object::Str(s) if name_eq(s, self.name()))
+        matches!(&key.0, Object::Str(s)
+            if crate::shared_value::SharedStr::ptr_eq(s, self.key()) || name_eq(s, self.name()))
     }
 }
 
@@ -541,7 +561,7 @@ fn lru_set(inst: &crate::types::PyInstance, field: LruField, v: Object) {
     match lru_slot_mut(&mut dict, field) {
         Some(slot) => *slot = v,
         None => {
-            dict.insert(DictKey(Object::from_static(field.name())), v);
+            dict.insert(field.key_object(), v);
         }
     }
 }
@@ -578,7 +598,7 @@ fn lru_cache_wrapper_new(args: &[Object]) -> Result<Object, RuntimeError> {
     {
         let mut dict = inst.dict_cell().borrow_mut();
         for (field, value) in LruField::ALL.into_iter().zip(values) {
-            dict.insert(DictKey(Object::from_static(field.name())), value);
+            dict.insert(field.key_object(), value);
         }
     }
     // CPython's `lru_cache_new` GC-tracks the wrapper: its `__wrapped__`
@@ -623,12 +643,17 @@ fn lru_make_key(call_args: &[Object], kwargs: &[(String, Object)], typed: bool)
 fn lru_counter_bump(inst: &crate::types::PyInstance, field: LruField) {
     // Keep the read and increment under one lock. Separate get/set borrows
     // lose updates when multiple threads hit a shared cache under gil=0.
-    let mut dict = inst.dict_cell().borrow_mut();
-    match lru_slot_mut(&mut dict, field) {
+    lru_bump(&mut inst.dict_cell().borrow_mut(), field);
+}
+
+/// Count one more of the counter `field` in the wrapper's `dict`.
+#[inline]
+fn lru_bump(dict: &mut DictData, field: LruField) {
+    match lru_slot_mut(dict, field) {
         Some(Object::Int(n)) => *n += 1,
         Some(count) => *count = Object::Int(1),
         None => {
-            dict.insert(DictKey(Object::from_static(field.name())), Object::Int(1));
+            dict.insert(field.key_object(), Object::Int(1));
         }
     }
 }
@@ -741,11 +766,11 @@ fn lru_state_links(state: &Object) -> Option<&Rc<RefCell<Vec<u8>>>> {
     }
 }
 
-fn lru_mode(dict: &DictData) -> Result<(bool, bool), RuntimeError> {
+fn lru_mode(state: Option<&Object>) -> Result<(bool, bool), RuntimeError> {
     // Read the tag without cloning active recency storage. The caller
     // releases its borrow before hashing the key or calling the wrapped
     // function.
-    match lru_slot(dict, LruField::State) {
+    match state {
         Some(Object::Bool(true)) => Ok((true, false)),
         Some(Object::Int(1)) => Ok((true, true)),
         Some(state @ Object::Tuple(_)) => {
@@ -769,7 +794,7 @@ fn scalar_lru_links(
     let mut dict = inst.dict_cell().borrow_mut();
     if lru_slot(&dict, LruField::State).is_none() {
         dict.insert(
-            DictKey(Object::from_static(LruField::State.name())),
+            LruField::State.key_object(),
             Object::Bool(false),
         );
     }
@@ -937,20 +962,113 @@ fn scalar_lru_operation(
     Ok(ScalarLru::Complete(None))
 }
 
+/// What [`lru_fast_lookup`] settled.
+enum FastLookup {
+    /// The cached value, its hit counted and its recency refreshed.
+    Hit(Object),
+    /// An exact miss, counted: the wrapped function to call (`None` if the
+    /// wrapper lost it).
+    Miss(Option<Object>),
+}
+
+/// The lookup of a native `key` in a bounded cache whose recency links are
+/// live, under one hold of the cache and its links: the hit or the counted
+/// miss, or `None` (nothing touched) for the general path to settle.
+#[inline]
+fn lru_fast_lookup(
+    inst: &crate::types::PyInstance,
+    cache: &RefCell<DictData>,
+    links: &RefCell<Vec<u8>>,
+    probe: &crate::object::LeafProbe<'_>,
+) -> Option<FastLookup> {
+    // The cache lock also serializes the links and the counters under
+    // gil=0 (taken in the order `lru_cache_clear` takes them). Nothing
+    // here runs Python.
+    let data = cache.borrow_mut();
+    let mut bytes = links.borrow_mut();
+    if bytes.is_empty() {
+        return None;
+    }
+    let mut order = lru_order::Order::new(&mut bytes, data.len())?;
+    match data.get_index_of(probe) {
+        Some(index) => {
+            let (stored, value) = data.get_index(index)?;
+            if !native_lru_key(&stored.0) {
+                return None;
+            }
+            order.touch(index)?;
+            let value = value.clone();
+            lru_bump(&mut inst.dict_cell().borrow_mut(), LruField::Hits);
+            Some(FastLookup::Hit(value))
+        }
+        None if probe.miss_is_exact() => {
+            let mut dict = inst.dict_cell().borrow_mut();
+            lru_bump(&mut dict, LruField::Misses);
+            Some(FastLookup::Miss(lru_slot(&dict, LruField::Wrapped).cloned()))
+        }
+        None => None,
+    }
+}
+
+/// Store `result` under the native `key` after [`lru_fast_lookup`]'s miss,
+/// evicting the least recent entry of a full cache in place: `Some` with
+/// the evicted entry (to release once the cache is consistent and
+/// unborrowed) when stored or already present, `None` (nothing touched)
+/// for the general path.
+fn lru_fast_store(
+    cache: &RefCell<DictData>,
+    links: &Rc<RefCell<Vec<u8>>>,
+    key: &Object,
+    result: &Object,
+    limit: usize,
+) -> Result<Option<Option<(DictKey, Object)>>, RuntimeError> {
+    let Some(probe) = crate::object::LeafProbe::new(key) else {
+        return Ok(None);
+    };
+    let mut data = cache.borrow_mut();
+    let mut bytes = links.borrow_mut();
+    let len = data.len();
+    if bytes.is_empty() || len > limit {
+        return Ok(None);
+    }
+    let Some(mut order) = lru_order::Order::new(&mut bytes, len) else {
+        return Ok(None);
+    };
+    match data.get_index_of(&probe) {
+        // A recursive call populated the key: its entry and recency win.
+        Some(_) => return Ok(Some(None)),
+        None if !probe.miss_is_exact() => return Ok(None),
+        None => {}
+    }
+    if len == limit {
+        // The least recent entry's place takes the new one, which becomes
+        // the most recent.
+        let Some(head) = order.head() else {
+            return Ok(None);
+        };
+        if order.touch(head).is_none() {
+            return Err(invalid_lru_order());
+        }
+        let evicted =
+            data.replace_index_unique(head, probe.hash, DictKey(key.clone()), result.clone());
+        return Ok(Some(evicted));
+    }
+    crate::object::bytearray_check_resizable(links)?;
+    if order.push(limit).is_none() {
+        return Err(invalid_lru_order());
+    }
+    data.insert_unique_hashed(probe.hash, DictKey(key.clone()), result.clone());
+    Ok(Some(None))
+}
+
 fn lru_call(args: &[Object], kwargs: &[(String, Object)]) -> Result<Object, RuntimeError> {
     let _depth_guard = lru_enter_native()?;
-    let inst = lru_self(args)?;
-    let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() else {
-        return Err(type_error("lru_cache requires a running interpreter"));
-    };
-    // SAFETY: published by the enclosing VM frame on this thread.
-    let interp = unsafe { &mut *ptr };
-    let func = || {
-        lru_get(&inst, LruField::Wrapped)
-            .ok_or_else(|| type_error("lru_cache wrapper lost its function"))
+    let Some(Object::Instance(inst)) = args.first() else {
+        return Err(type_error("expected _lru_cache_wrapper instance"));
     };
-    // One borrow reads the size, the mode, and the cache.
-    let (maxsize, mode, cache) = {
+    let call_args = &args[1..];
+    // One borrow reads the size, the mode, the cache, and live links.
+    let (maxsize, mode, cache, links) = {
         let dict = inst.dict_cell().borrow();
         let cache = match lru_slot(&dict, LruField::Cache) {
             Some(Object::Dict(cache)) => Some(cache.clone()),
@@ -959,19 +1077,59 @@ fn lru_call(args: &[Object], kwargs: &[(String, Object)]) -> Result<Object, Runt
         let maxsize = lru_slot(&dict, LruField::Maxsize)
             .cloned()
             .unwrap_or(Object::None);
-        (maxsize, lru_mode(&dict), cache)
+        let state = lru_slot(&dict, LruField::State);
+        let links = state.and_then(lru_state_links).cloned();
+        (maxsize, lru_mode(state), cache, links)
+    };
+    let interp = || match crate::vm_singletons::current_interpreter_ptr() {
+        // SAFETY: published by the enclosing VM frame on this thread.
+        Some(ptr) => Ok(unsafe { &mut *ptr }),
+        None => Err(type_error("lru_cache requires a running interpreter")),
+    };
+    let func = || {
+        lru_get(inst, LruField::Wrapped)
+            .ok_or_else(|| type_error("lru_cache wrapper lost its function"))
     };
-    let call_args = &args[1..];
-    let globals = interp.builtins_dict();
 
     // maxsize == 0: no caching, statistics only.
     if matches!(maxsize, Object::Int(0)) {
-        lru_counter_bump(&inst, LruField::Misses);
+        let interp = interp()?;
+        lru_counter_bump(inst, LruField::Misses);
+        let globals = interp.builtins_dict();
         return interp.call(&func()?, call_args, kwargs, &globals);
     }
 
     let (typed, scalar_enabled) = mode?;
     let key = lru_make_key(call_args, kwargs, typed);
+    // The warm shape: an `int`, `str`, or tuple of them in a bounded cache
+    // whose recency links are live.
+    if let (Object::Int(limit @ 1..), Some(cache), Some(links), true) =
+        (&maxsize, &cache, &links, scalar_enabled)
+    {
+        let probe = native_lru_key(&key)
+            .then(|| crate::object::LeafProbe::new(&key))
+            .flatten();
+        let found = probe.and_then(|p| lru_fast_lookup(inst, cache, links, &p));
+        match found {
+            Some(FastLookup::Hit(v)) => return Ok(v),
+            Some(FastLookup::Miss(func)) => {
+                let func = func.ok_or_else(|| type_error("lru_cache wrapper lost its function"))?;
+                let interp = interp()?;
+                let globals = interp.builtins_dict();
+                let result = interp.call(&func, call_args, kwargs, &globals)?;
+                let limit = usize::try_from(*limit).unwrap_or(usize::MAX);
+                if let Some(evicted) = lru_fast_store(cache, links, &key, &result, limit)? {
+                    // Released with the cache consistent and unborrowed.
+                    drop(evicted);
+                    return Ok(result);
+                }
+                return lru_store(inst, cache, &key, result, &maxsize, true);
+            }
+            None => {}
+        }
+    }
+    let interp = interp()?;
+    let globals = interp.builtins_dict();
     // Compute a tuple key's hash before touching the cache. Successful
     // hashes are retained by the tuple, so lookup, recency updates, and
     // insertion invoke each argument's hash only once per call. Errors
@@ -986,7 +1144,7 @@ fn lru_call(args: &[Object], kwargs: &[(String, Object)]) -> Result<Object, Runt
     let bounded = matches!(maxsize, Object::Int(m) if m > 0);
 
     let scalar_hit = if bounded && scalar_enabled {
-        scalar_lru_operation(&inst, &cache, &key, None)?
+        scalar_lru_operation(inst, &cache, &key, None)?
     } else {
         ScalarLru::Slow
     };
@@ -1013,30 +1171,43 @@ fn lru_call(args: &[Object], kwargs: &[(String, Object)]) -> Result<Object, Runt
         ),
     };
     if let Some(v) = hit {
-        lru_counter_bump(&inst, LruField::Hits);
+        lru_counter_bump(inst, LruField::Hits);
         return Ok(v);
     }
-    lru_counter_bump(&inst, LruField::Misses);
+    lru_counter_bump(inst, LruField::Misses);
     let result = interp.call(&func()?, call_args, kwargs, &globals)?;
+    lru_store(inst, &cache, &key, result, &maxsize, scalar)
+}
+
+/// Store a miss's `result` under `key` (through the scalar links when
+/// `scalar`), unless a reentrant call stored the key first.
+fn lru_store(
+    inst: &crate::types::PyInstance,
+    cache: &Rc<RefCell<DictData>>,
+    key: &Object,
+    result: Object,
+    maxsize: &Object,
+    scalar: bool,
+) -> Result<Object, RuntimeError> {
     if scalar {
-        let Object::Int(limit) = maxsize else {
+        let Object::Int(limit) = *maxsize else {
             unreachable!()
         };
         let limit = usize::try_from(limit).unwrap_or(usize::MAX);
         if matches!(
-            scalar_lru_operation(&inst, &cache, &key, Some((&result, limit)))?,
+            scalar_lru_operation(inst, cache, key, Some((&result, limit)))?,
             ScalarLru::Complete(_)
         ) {
             return Ok(result);
         }
     }
     crate::object::key_cmp_scope(|| {
-        with_stolen_cache(&cache, |c| {
+        with_stolen_cache(cache, |c| {
             // A reentrant call may have populated the key while `func` ran;
             // keep the existing entry (CPython does the same).
             if !c.contains_key(&DictKey(key.clone())) {
                 c.insert(DictKey(key.clone()), result.clone());
-                if let Object::Int(m) = maxsize {
+                if let Object::Int(m) = *maxsize {
                     while c.len() > m.max(0) as usize {
                         c.shift_remove_index(0);
                     }
diff --git a/crates/weavepy-vm/src/stdlib/functools_mod/lru_order.rs b/crates/weavepy-vm/src/stdlib/functools_mod/lru_order.rs
index 1db4651c..9999cdda 100644
--- a/crates/weavepy-vm/src/stdlib/functools_mod/lru_order.rs
+++ b/crates/weavepy-vm/src/stdlib/functools_mod/lru_order.rs
@@ -20,14 +20,18 @@ pub(super) fn empty() -> Vec<u8> {
 pub(super) struct Order<'a> {
     bytes: &'a mut Vec<u8>,
     len: usize,
+    /// The words the buffer holds: the header's two and two per node.
+    words: usize,
 }
 
 impl<'a> Order<'a> {
+    #[inline]
     pub(super) fn new(bytes: &'a mut Vec<u8>, len: usize) -> Option<Self> {
-        if bytes.len() != len.checked_add(1)?.checked_mul(2 * WORD)? {
+        let words = len.checked_add(1)?.checked_mul(2)?;
+        if bytes.len() != words.checked_mul(WORD)? {
             return None;
         }
-        let order = Self { bytes, len };
+        let order = Self { bytes, len, words };
         let head = order.word(HEAD)?;
         let tail = order.word(TAIL)?;
         if (len == 0 && (head != NONE || tail != NONE))
@@ -38,24 +42,33 @@ impl<'a> Order<'a> {
         Some(order)
     }
 
+    #[inline(always)]
     fn word(&self, index: usize) -> Option<usize> {
-        let start = index.checked_mul(WORD)?;
-        Some(usize::from_ne_bytes(
-            self.bytes
-                .get(start..start.checked_add(WORD)?)?
-                .try_into()
-                .ok()?,
-        ))
+        if index >= self.words {
+            return None;
+        }
+        // SAFETY: the buffer holds `words` whole words (checked by `new`,
+        // kept by `push` and `swap_remove`).
+        Some(unsafe { self.bytes.as_ptr().cast::<usize>().add(index).read_unaligned() })
     }
 
+    #[inline(always)]
     fn set_word(&mut self, index: usize, value: usize) -> Option<()> {
-        let start = index.checked_mul(WORD)?;
-        self.bytes
-            .get_mut(start..start.checked_add(WORD)?)?
-            .copy_from_slice(&value.to_ne_bytes());
+        if index >= self.words {
+            return None;
+        }
+        // SAFETY: as in `word`.
+        unsafe {
+            self.bytes
+                .as_mut_ptr()
+                .cast::<usize>()
+                .add(index)
+                .write_unaligned(value);
+        }
         Some(())
     }
 
+    #[inline(always)]
     fn node(&self, index: usize) -> Option<(usize, usize)> {
         if index >= self.len {
             return None;
@@ -63,6 +76,7 @@ impl<'a> Order<'a> {
         Some((self.word(2 + 2 * index)?, self.word(3 + 2 * index)?))
     }
 
+    #[inline]
     fn detach(&mut self, index: usize) -> Option<()> {
         let (prev, next) = self.node(index)?;
         if prev == index || next == index {
@@ -93,6 +107,7 @@ impl<'a> Order<'a> {
         (head != NONE).then_some(head)
     }
 
+    #[inline]
     pub(super) fn touch(&mut self, index: usize) -> Option<()> {
         if index >= self.len {
             return None;
@@ -127,6 +142,7 @@ impl<'a> Order<'a> {
         self.bytes.extend_from_slice(&tail.to_ne_bytes());
         self.bytes.extend_from_slice(&NONE.to_ne_bytes());
         self.len += 1;
+        self.words += 2;
         if tail == NONE {
             self.set_word(HEAD, index)?;
         } else {
@@ -162,7 +178,8 @@ impl<'a> Order<'a> {
             }
         }
         self.len -= 1;
-        self.bytes.truncate((self.len + 1) * 2 * WORD);
+        self.words -= 2;
+        self.bytes.truncate(self.words * WORD);
         Some(())
     }
 

From 73826e3b6aaec86460f1ae6a39e9b39a11887017 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 18:39:15 -0700
Subject: [PATCH 116/173] perf: keep decorator wrappers in compiled code

A decorator's wrapper that counts its calls and forwards its arguments
(`wrapper.calls += 1; return fn(*args, **kwargs)`) left its compiled body
at the attribute store on every call, and the core loop ran the rest. The
frame JIT now:

- reads and writes a function's own attributes in its attribute helpers,
  through the core loop's function load and store (non-dunder names in
  the function's `__dict__`);
- runs the forwarding shape `BUILD_MAP 0; LOAD_FAST kwargs; DICT_MERGE 1;
  CALL_FUNCTION_EX` through a new helper, h_call_forward, which binds the
  call from the local mapping as the core loop's `core_call_forward` does
  and runs a compiled callee directly; a decline leaves at the BUILD_MAP.

With both, the wrapper and the wrapped function return directly instead
of exiting to the core loop.

Measured (instructions per iteration, macOS, pinned hash seed): a
`functools.wraps` wrapper forwarding two calls with keywords, 14,920 ->
11,634 (CPython 3.14: 7,406).

Adds a regression test for function attributes and forwarded keyword
binding in warm code.
---
 crates/weavepy-vm/src/frame_jit.rs            | 124 +++++++++++++++-
 .../test_function_attrs_forwarding.py         | 135 ++++++++++++++++++
 2 files changed, 257 insertions(+), 2 deletions(-)
 create mode 100644 tests/regrtest/test_function_attrs_forwarding.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index e7ce963d..0df96f26 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -584,6 +584,10 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool
         if helped[k] || matches!(op, OpCode::Call | OpCode::CallKw | OpCode::CallEx) {
             return Some(1);
         }
+        // A forwarding call's merge runs with its call (`call_forward`).
+        if op == OpCode::DictUpdate && k >= 2 && crate::forward_call_shape(code, k - 2) {
+            return Some(1);
+        }
         if !native_at(code, k) {
             return None;
         }
@@ -3755,6 +3759,8 @@ unsafe extern "C" fn h_stack_attr(
                 crate::drop_hot(std::mem::replace(&mut *at, v));
                 return 0;
             }
+            // A function's stored attribute (`wrapper.calls`).
+            Object::Function(_) => return u32::from(!Interpreter::core_function_load(code, arg, at)),
             _ => None,
         };
         let Some(v) = v else {
@@ -3861,8 +3867,14 @@ unsafe extern "C" fn h_stack_store_attr(code: *const CodeObject, pc: u64, at: *m
     unsafe {
         let (code, pc) = (&*code, pc as usize);
         let (val, recv) = (&*at, &*at.add(1));
-        let Object::Instance(inst) = recv else {
-            return 1;
+        let inst = match recv {
+            Object::Instance(inst) => inst,
+            // A function's attribute (`wrapper.calls += 1`): both leave.
+            Object::Function(_) => {
+                let arg = code.instructions[pc].arg;
+                return u32::from(!Interpreter::core_function_store(code, arg, at.add(1)));
+            }
+            _ => return 1,
         };
         if !droppable(recv)
             || !Interpreter::core_store_attr(code, inst, pc, code.instructions[pc].arg, val)
@@ -5801,6 +5813,58 @@ unsafe extern "C" fn h_call_ex(st: *mut State, code: *const CodeObject, pc: u64,
     }
 }
 
+/// `f(*args, **kwargs)`'s forwarding run (see `forward_call_shape`)
+/// opening at the `BUILD_MAP` at `pc` on a stack `len` deep, as the core
+/// loop's arm runs it (`Interpreter::core_call_forward`), the frame synced
+/// first: `0` the call's result in place (as [`h_call_ex`]'s), `RELOAD`
+/// the core loop goes on, `CALL_DECLINED` untouched.
+unsafe extern "C" fn h_call_forward(
+    st: *mut State,
+    code: *const CodeObject,
+    pc: u64,
+    len: u64,
+) -> u32 {
+    // SAFETY: the code passes its live state, its own code, one of its
+    // forwarding `BUILD_MAP`s and its stack depth.
+    unsafe {
+        let _ = code;
+        let st = &mut *st;
+        let (pc, len) = (pc as usize, len as usize);
+        if st.sw.is_null() || len < 3 {
+            return CALL_DECLINED;
+        }
+        let start = len - 3;
+        let frame = &mut *st.frame;
+        frame.stack.set_len(len);
+        frame.pc = pc as u32;
+        let sw = &mut *st.sw;
+        *sw.last = st.last;
+        let saved_last = sw.last;
+        let interp = &mut *st.interp.cast_mut();
+        if !interp.core_call_forward(sw, pc) {
+            return if sw.pending.is_some() {
+                RELOAD
+            } else {
+                CALL_DECLINED
+            };
+        }
+        if sw.pending.is_none() && !std::ptr::eq(sw.cur, st.frame) {
+            // (The call's own pc, three on, is the caller's last.)
+            if let Some(r) = run_pushed_directly(st, pc + 3, start, saved_last) {
+                return r;
+            }
+            if run_callee_directly(st, start) {
+                st.len = start + 1;
+                return 0;
+            }
+        }
+        if finished_in_place(st, start) {
+            return 0;
+        }
+        RELOAD
+    }
+}
+
 /// `RETURN_VALUE` (the `pc`th instruction) of an inline activation with
 /// the value atop a `len`-deep stack: the frame synced and the caller
 /// switched back to as the core loop's arm does (`RELOAD`), the state
@@ -7443,6 +7507,15 @@ impl<'a> Lower<'a> {
             {
                 starts[pc + 2] = true;
             }
+            // A forwarding call (`f(*args, **kwargs)`) continues after its
+            // `CALL_FUNCTION_EX`.
+            if ins.op == OpCode::BuildMap
+                && ins.arg == 0
+                && crate::forward_call_shape(code, pc)
+                && pc + 4 < n
+            {
+                starts[pc + 4] = true;
+            }
             // A global's fused class-constant read continues after its
             // `LOAD_ATTR`.
             if ins.op == OpCode::LoadGlobal
@@ -7842,6 +7915,11 @@ impl<'a> Lower<'a> {
                 return self.build(pc, ins.arg as usize, false)
             }
             OpCode::BuildList => return self.build(pc, ins.arg as usize, true),
+            OpCode::BuildMap
+                if ins.arg == 0 && !self.is_gen() && crate::forward_call_shape(self.code, pc) =>
+            {
+                return self.call_forward(pc)
+            }
             OpCode::BuildMap if (0..=8).contains(&ins.arg) => {
                 return self.build_map(pc, ins.arg as usize)
             }
@@ -10474,6 +10552,48 @@ impl<'a> Lower<'a> {
         self.call_with(pc, h_call_ex as *const () as usize)
     }
 
+    /// `f(*args, **kwargs)`'s forwarding run, `BUILD_MAP 0` at `pc` through
+    /// its `CALL_FUNCTION_EX`, through [`h_call_forward`] (as
+    /// [`Self::call_with`]): the block goes on past the call; a decline
+    /// leaves at the `BUILD_MAP`, where the core loop runs it.
+    fn call_forward(&mut self, pc: usize) -> bool {
+        let after = self.depths.get(pc + 4).copied().unwrap_or(-1);
+        if after < 0 || self.depth < 3 || self.blocks.get(pc + 4).copied().flatten().is_none() {
+            return self.build_map(pc, 0);
+        }
+        self.flush();
+        self.store_last();
+        let code = self.code_ptr();
+        let pcv = self.b.ins().iconst(types::I64, pc as i64);
+        let len = self.b.ins().iconst(types::I64, self.depth as i64);
+        let r = self
+            .call(
+                h_call_forward as *const () as usize,
+                &[self.st, code, pcv, len],
+                true,
+            )
+            .expect("returns");
+        let out = self.exit_with(pc, &[], INTERP);
+        let ran = self.b.create_block();
+        let other = self.b.create_block();
+        self.b.ins().brif(r, other, &[], ran, &[]);
+        self.b.switch_to_block(other);
+        let declined = self
+            .b
+            .ins()
+            .icmp_imm(IntCC::Equal, r, i64::from(CALL_DECLINED));
+        let leave = self.b.create_block();
+        self.b.ins().brif(declined, out, &[], leave, &[]);
+        self.b.switch_to_block(leave);
+        self.leave(r);
+        self.b.switch_to_block(ran);
+        self.depth = after as usize;
+        self.set_last(pc + 3);
+        self.check_released(pc + 4);
+        self.goto(pc + 4);
+        false
+    }
+
     /// A call instruction through `helper` (`h_call_kw`'s signature): the
     /// result in place when it ran, anything else the core loop's.
     fn call_with(&mut self, pc: usize, helper: usize) -> bool {
diff --git a/tests/regrtest/test_function_attrs_forwarding.py b/tests/regrtest/test_function_attrs_forwarding.py
new file mode 100644
index 00000000..70849091
--- /dev/null
+++ b/tests/regrtest/test_function_attrs_forwarding.py
@@ -0,0 +1,135 @@
+"""Function attributes and `f(*args, **kwargs)` forwarding in hot code.
+
+Decorators count calls on the wrapper (`wrapper.calls += 1`) and forward
+every argument to the wrapped function. Compiled code reads and writes a
+function's `__dict__` directly and binds the forwarded mapping without
+building the merged copy; these checks keep both exact once the loops are
+warm.
+"""
+
+import functools
+
+N = 3000
+
+
+def counted(fn):
+    @functools.wraps(fn)
+    def wrapper(*args, **kwargs):
+        wrapper.calls += 1
+        return fn(*args, **kwargs)
+
+    wrapper.calls = 0
+    return wrapper
+
+
+@counted
+def area(w, h=1, *, scale=1.0):
+    return w * h * scale
+
+
+total = 0.0
+for i in range(N):
+    total += area(i, h=2, scale=0.5)
+    total += area(3)
+assert area.calls == 2 * N, area.calls
+assert total == sum(i * 2 * 0.5 + 3 for i in range(N)), total
+assert area.__wrapped__.__name__ == 'area' and area.__name__ == 'area'
+assert 'calls' in area.__dict__ and '__wrapped__' in area.__dict__
+
+
+# A replaced `__dict__` is the one compiled reads and writes see.
+def bump(f, n):
+    for _ in range(n):
+        f.hits += 1
+    return f.hits
+
+
+def target():
+    pass
+
+
+target.hits = 0
+assert bump(target, N) == N
+fresh = {'hits': 100}
+target.__dict__ = fresh
+assert bump(target, 5) == 105 and fresh['hits'] == 105
+assert target.__dict__ is fresh
+
+
+# A deleted attribute raises AttributeError from the compiled read.
+del target.hits
+try:
+    bump(target, 1)
+except AttributeError as e:
+    assert 'hits' in str(e), e
+else:
+    raise AssertionError('missing function attribute read succeeded')
+
+
+# Dunder names keep the function type's descriptors.
+def named():
+    pass
+
+
+def rename(f, n):
+    for k in range(n):
+        f.__qualname__ = 'q%d' % k
+        f.__doc__ = 'd%d' % k
+    return f.__qualname__, f.__doc__, f.__dict__
+
+
+assert rename(named, N) == ('q%d' % (N - 1), 'd%d' % (N - 1), {})
+
+
+# Forwarded keywords bind by name, and a callee's own `**kw` is a copy.
+def collect(a, b=2, **kw):
+    kw['seen'] = True
+    return a, b, kw
+
+
+def forward(*args, **kwargs):
+    return collect(*args, **kwargs)
+
+
+for i in range(N):
+    kwargs = {'b': i, 'extra': i + 1}
+    a, b, kw = forward(i, **kwargs)
+    assert (a, b, kw) == (i, i, {'extra': i + 1, 'seen': True})
+    assert kwargs == {'b': i, 'extra': i + 1}
+
+
+# Errors from the forwarded binding are the callee's.
+def strict(x, *, flag=False):
+    return x, flag
+
+
+def forward_strict(*args, **kwargs):
+    return strict(*args, **kwargs)
+
+
+for i in range(N):
+    assert forward_strict(i, flag=True) == (i, True)
+for bad, message in (
+    (lambda: forward_strict(1, x=2), "got multiple values for argument 'x'"),
+    (lambda: forward_strict(1, nope=2), "got an unexpected keyword argument 'nope'"),
+    (lambda: forward_strict(), "missing 1 required positional argument: 'x'"),
+    (lambda: forward_strict(1, 2), 'takes 1 positional argument but 2 were given'),
+):
+    try:
+        bad()
+    except TypeError as e:
+        assert message in str(e), (message, str(e))
+    else:
+        raise AssertionError(message)
+
+
+# A forwarded call into a builtin and into a class.
+def call_through(f, *args, **kwargs):
+    return f(*args, **kwargs)
+
+
+for i in range(N):
+    assert call_through(max, i, 3, key=lambda v: -v) == min(i, 3)
+    assert call_through(dict, a=i) == {'a': i}
+
+print('function attributes and forwarding: ok')

From eecaba20435ca486e400a1a548a427234cd020aa Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 19:33:44 -0700
Subject: [PATCH 117/173] perf: build a function with its defaults and closure
 at once

A `def` or `lambda` that runs per call (`make_adder(i)`, a sort key)
compiles to a bare MAKE_FUNCTION followed by SET_FUNCTION_ATTRIBUTEs for
its defaults and closure. The frame JIT ran each one through its own
helper, and each attribute store untracked the fresh function from the
cycle collector, rebuilt it in place, and tracked it again. The frame JIT
now builds the function with its defaults and closure in one helper,
h_make_function_attrs, as the core loop's arm already does.

Making a function also got cheaper on its own:
- `__module__` is read from a module namespace's first entry (its
  `__name__`) before falling back to the hashed lookup;
- the function's name shares the code object's pooled name instead of
  copying it into a new String (and freeing it again on death).

Measured (instructions per iteration, macOS, pinned hash seed):
- a closure made per iteration and applied with a lambda: 7,973 ->
  7,192 (CPython 3.14: 4,758)
- sorted() of three ints with a key lambda: 7,360 -> 7,114 (CPython
  3.14: 5,151)

Adds a regression test for functions and closures made in warm code.
---
 crates/weavepy-vm/src/frame_jit.rs           | 117 +++++++++++++++++++
 crates/weavepy-vm/src/lib.rs                 |  16 ++-
 crates/weavepy-vm/src/object.rs              |   2 +-
 crates/weavepy-vm/src/stdlib/weakref_real.rs |   2 +-
 tests/regrtest/test_closure_creation_hot.py  | 105 +++++++++++++++++
 5 files changed, 234 insertions(+), 8 deletions(-)
 create mode 100644 tests/regrtest/test_closure_creation_hot.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 0df96f26..d9d896d7 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -6286,6 +6286,80 @@ unsafe extern "C" fn h_make_function(st: *mut State, at: *mut Object, flags: u64
     }
 }
 
+/// How many `SET_FUNCTION_ATTRIBUTE`s right after the bare `MAKE_FUNCTION`
+/// at `pc` attach defaults (`0x01`) or a closure (`0x08`), each once:
+/// the ones [`h_make_function_attrs`] builds the function with.
+fn fused_function_attrs(code: &CodeObject, pc: usize) -> usize {
+    let ins = &code.instructions;
+    let mut seen = 0;
+    let mut k = 0;
+    while let Some(next) = ins.get(pc + 1 + k) {
+        let flag = next.arg;
+        if next.op != OpCode::SetFunctionAttribute
+            || !matches!(flag, 0x01 | 0x08)
+            || seen & flag != 0
+        {
+            break;
+        }
+        seen |= flag;
+        k += 1;
+    }
+    k
+}
+
+/// The bare `MAKE_FUNCTION` at `pc` of `code` with the `k`
+/// `SET_FUNCTION_ATTRIBUTE`s after it (see [`fused_function_attrs`]), the
+/// code object at stack slot `at + k` and the first attribute's value next
+/// below it, as the core loop's arm runs them: `0` the function at `at`
+/// with its defaults and closure (the operands consumed), anything else
+/// declined untouched.
+unsafe extern "C" fn h_make_function_attrs(
+    st: *mut State,
+    code: *const CodeObject,
+    pc: u64,
+    at: *mut Object,
+    k: u64,
+) -> u32 {
+    // SAFETY: the code passes its live state, its own code, the fused run
+    // at `pc`, and its `k + 1` initialized operand slots.
+    unsafe {
+        let (code, pc, k) = (&*code, pc as usize, k as usize);
+        if crate::capi_watchers::funcs_active() || !matches!(&*at.add(k), Object::Code(_)) {
+            return 1;
+        }
+        let mut defaults = Vec::new();
+        let mut closure = Vec::new();
+        for j in 0..k {
+            let Object::Tuple(items) = &*at.add(k - 1 - j) else {
+                return 1;
+            };
+            match code.instructions[pc + 1 + j].arg {
+                0x01 => defaults = items.iter().cloned().collect(),
+                _ => closure = items.iter().cloned().collect(),
+            }
+        }
+        let Object::Code(fn_code) = at.add(k).read() else {
+            unreachable!("checked above")
+        };
+        // (The tuples' items live on in the function.)
+        for j in 0..k {
+            drop(at.add(j).read());
+        }
+        let frame = &*(*st).frame;
+        let f = crate::new_function(
+            fn_code,
+            &frame.globals,
+            &frame.builtins,
+            defaults,
+            Vec::new(),
+            closure,
+            None,
+        );
+        at.write(f);
+        0
+    }
+}
+
 /// `SET_FUNCTION_ATTRIBUTE flag` of the value at stack slot `at` on the
 /// fresh function above it: `0` the function at `at`, `RAISED` its error
 /// (`st` synced past the instruction at `pc`), anything else declined
@@ -7507,6 +7581,14 @@ impl<'a> Lower<'a> {
             {
                 starts[pc + 2] = true;
             }
+            // A function built with its defaults or closure continues after
+            // its last `SET_FUNCTION_ATTRIBUTE`.
+            if ins.op == OpCode::MakeFunction && ins.arg == 0 {
+                let k = fused_function_attrs(code, pc);
+                if k > 0 && pc + 1 + k < n {
+                    starts[pc + 1 + k] = true;
+                }
+            }
             // A forwarding call (`f(*args, **kwargs)`) continues after its
             // `CALL_FUNCTION_EX`.
             if ins.op == OpCode::BuildMap
@@ -7977,6 +8059,41 @@ impl<'a> Lower<'a> {
                 self.check_released(pc + 1);
                 true
             }
+            // A `def` or `lambda` with its defaults or closure: the
+            // function built with them at once.
+            OpCode::MakeFunction
+                if ins.arg == 0
+                    && fused_function_attrs(self.code, pc) > 0
+                    && self.depth > fused_function_attrs(self.code, pc) =>
+            {
+                let k = fused_function_attrs(self.code, pc);
+                if self.blocks.get(pc + 1 + k).copied().flatten().is_none() {
+                    self.exit(INTERP, pc);
+                    return false;
+                }
+                self.flush();
+                let s = self.depth - (k + 1);
+                let at = self.slot_addr(s);
+                let code = self.code_ptr();
+                let pcv = self.b.ins().iconst(types::I64, pc as i64);
+                let kv = self.b.ins().iconst(types::I64, k as i64);
+                let r = self
+                    .call(
+                        h_make_function_attrs as *const () as usize,
+                        &[self.st, code, pcv, at, kv],
+                        true,
+                    )
+                    .expect("returns");
+                let out = self.exit_with(pc, &[], INTERP);
+                self.branch_out(r, out);
+                for _ in 0..=k {
+                    self.pop();
+                }
+                self.push(Item::Mem(s));
+                self.set_last(pc + k);
+                self.goto(pc + 1 + k);
+                false
+            }
             OpCode::MakeFunction | OpCode::SetFunctionAttribute => {
                 let n = if ins.op == OpCode::MakeFunction {
                     1 + (ins.arg & 0xf).count_ones() as usize
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 261f5065..ff7bbcf7 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -52941,7 +52941,7 @@ impl Interpreter {
                 .unwrap_or_else(|| self.builtins.clone()),
         };
         Ok(Object::Function(Rc::new(crate::object::PyFunction {
-            name,
+            name: name.into(),
             code: RefCell::new(code),
             globals,
             builtins,
@@ -75270,12 +75270,16 @@ fn new_function(
     // staticmethod wrappers) return the *same* object, which
     // `assertIs(wrapper.__name__, func.__name__)` in test_decorators
     // relies on.
-    let module = globals
-        .borrow()
-        .get(&DictKey(fn_slot_keys()[1].clone()))
-        .cloned();
+    let module = {
+        let g = globals.borrow();
+        match g.get_index(0) {
+            // A module's namespace starts with its `__name__`.
+            Some((k, v)) if matches!(&k.0, Object::Str(s) if &**s == "__name__") => Some(v.clone()),
+            _ => g.get(&DictKey(fn_slot_keys()[1].clone())).cloned(),
+        }
+    };
     let f = PyFunction {
-        name: code.name.to_string(),
+        name: code.name.clone(),
         code: RefCell::new(code),
         globals: globals.clone(),
         // The defining frame's builtins, not a fresh resolution: the same
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index b95db8b1..3a40b6c2 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -4981,7 +4981,7 @@ impl<'a> IntoIterator for &'a mut DictData {
 
 #[derive(Clone)]
 pub struct PyFunction {
-    pub name: String,
+    pub name: weavepy_compiler::Name,
     /// The code object executed on call. Interior-mutable because
     /// `f.__code__ = other.__code__` rebinds it at runtime (CPython's
     /// `func_set_code`); every call reads the *current* value.
diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs
index d83006fb..4e03fa36 100644
--- a/crates/weavepy-vm/src/stdlib/weakref_real.rs
+++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs
@@ -382,7 +382,7 @@ fn fq_type_name(target: &Object) -> String {
 fn referent_display_name(target: &Object) -> Option<String> {
     match target {
         Object::Type(t) => Some(t.name.clone()),
-        Object::Function(f) => Some(f.name.clone()),
+        Object::Function(f) => Some(f.name.to_string()),
         Object::Module(m) => Some(m.name.clone()),
         Object::Instance(i) => match i.cls().lookup("__name__")? {
             Object::Str(s) => Some(s.to_string()),
diff --git a/tests/regrtest/test_closure_creation_hot.py b/tests/regrtest/test_closure_creation_hot.py
new file mode 100644
index 00000000..a8488570
--- /dev/null
+++ b/tests/regrtest/test_closure_creation_hot.py
@@ -0,0 +1,105 @@
+"""Functions and closures made per call in warm code.
+
+A `def` or `lambda` run in a loop is built with its defaults and closure
+at once; its cells, defaults, names, module and late binding must match a
+function built one attribute at a time.
+"""
+
+import gc
+import weakref
+
+N = 3000
+
+
+def make_adder(k):
+    def add(x, scale=2):
+        return x * scale + k
+    return add
+
+
+def make_counter():
+    count = 0
+
+    def bump(step=1):
+        nonlocal count
+        count += step
+        return count
+    return bump
+
+
+def late_binding(n):
+    fns = []
+    for i in range(n):
+        fns.append(lambda: i)
+    return [f() for f in fns]
+
+
+total = 0
+for i in range(N):
+    add = make_adder(i)
+    total += add(1)
+    assert add.__defaults__ == (2,)
+    assert add.__closure__[0].cell_contents == i
+    assert add.__name__ == 'add'
+    assert add.__qualname__ == 'make_adder.<locals>.add'
+    assert add.__module__ == __name__
+    assert add.__kwdefaults__ is None
+assert total == sum(2 + i for i in range(N)), total
+
+counter = make_counter()
+for i in range(N):
+    assert counter() == i + 1
+assert counter(10) == N + 10
+assert counter.__closure__[0].cell_contents == N + 10
+
+assert late_binding(5) == [4] * 5
+
+
+def with_kwdefaults(i):
+    def f(a, b=i, *, c=i + 1):
+        return a, b, c
+    return f
+
+
+for i in range(N):
+    f = with_kwdefaults(i)
+    assert f(0) == (0, i, i + 1)
+    assert f.__kwdefaults__ == {'c': i + 1}
+
+
+# A closure's function dies with its last reference, and a cycle through
+# its cell is collected.
+class Sentinel:
+    pass
+
+
+def capture(obj):
+    def inner():
+        return obj
+    return inner
+
+
+refs = []
+for i in range(100):
+    s = Sentinel()
+    refs.append(weakref.ref(s))
+    fn = capture(s)
+    assert fn() is s
+    del s, fn
+assert all(r() is None for r in refs)
+
+
+def cyclic():
+    def me():
+        return me
+    return me
+
+
+for i in range(N):
+    f = cyclic()
+    assert f() is f
+w = weakref.ref(f)
+del f
+gc.collect()
+assert w() is None
+print('closure creation: ok')

From 6c37c3a923f6f676091435ca54b12df5d978a77c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 20:00:31 -0700
Subject: [PATCH 118/173] perf: read and write function attributes with the
 name's stored hash

`wrapper.calls += 1` on a function read the attribute through the general
leaf read (a slot-name check, two cell borrows) and wrote it by probing
the function's `__dict__` with a key that hashed the name again. Both now
probe with the code object's interned name and its stored hash, and read
the function's dictionary cell in place: a plain (non-dunder) name's load
reads the `__dict__` entry directly, and the store finds the existing
dictionary without cloning its handle. Dunder names and cells in use keep
the existing paths.

Measured (instructions per iteration, macOS, pinned hash seed): a
`functools.wraps` wrapper counting and forwarding two calls, 11,566 ->
11,192 (CPython 3.14: 7,406).
---
 crates/weavepy-vm/src/lib.rs | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ff7bbcf7..fd59a449 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -18973,7 +18973,18 @@ impl Interpreter {
         let Object::Function(f) = (unsafe { &*top }) else {
             return false;
         };
-        let Some(v) = Self::leaf_function_read(code, f, name_idx) else {
+        // A plain name (no `function` descriptor is one) is the
+        // `__dict__` entry, probed with the name's interned bytes and
+        // stored hash; anything else, or a cell in use, takes the full read.
+        let plain = code_name_key(code, name_idx)
+            .filter(|key| !key.s.starts_with("__") && !crate::object::exotic_str_keys_possible())
+            .and_then(|key| {
+                // SAFETY: reads between two instructions, running no code;
+                // the views end with the value's clone.
+                let attrs = unsafe { f.attrs.peek() }?.as_ref()?;
+                unsafe { attrs.peek() }?.get(&key).map(Self::clone_operand)
+            });
+        let Some(v) = plain.or_else(|| Self::leaf_function_read(code, f, name_idx)) else {
             return false;
         };
         // SAFETY: as above; the function is held by the variable it was
@@ -19006,13 +19017,25 @@ impl Interpreter {
         {
             return false;
         }
-        let Some(name @ Object::Str(n)) = code_name_obj(code, name_idx) else {
+        let (Some(name @ Object::Str(n)), Some(key)) =
+            (code_name_obj(code, name_idx), code_name_key(code, name_idx))
+        else {
             return false;
         };
         if n.starts_with("__") {
             return false;
         }
-        let dict = f.attrs();
+        // The existing dictionary in place (no code runs while it is
+        // read), else a new one.
+        let made;
+        // SAFETY: a read of the cell that holds the dictionary.
+        let dict = match unsafe { f.attrs.peek() } {
+            Some(Some(d)) => d,
+            _ => {
+                made = f.attrs();
+                &made
+            }
+        };
         let old = {
             let Ok(mut d) = dict.try_borrow_mut() else {
                 return false;
@@ -19020,7 +19043,8 @@ impl Interpreter {
             // SAFETY: the value moves into the dictionary; its slot is
             // forgotten below.
             let value = unsafe { std::ptr::read(value) };
-            match d.get_mut(&crate::object::StrKey(n)) {
+            // (Probed with the name's interned bytes and stored hash.)
+            match d.get_mut(&key) {
                 Some(slot) => Some(std::mem::replace(slot, value)),
                 None => {
                     d.insert(DictKey(name.clone()), value);

From 70a28abf5169c45d5991bb636a1b1e6b84a41d59 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 17:08:33 -0700
Subject: [PATCH 119/173] perf: call a compiled site's settled builtin through
 its own lean helper

A compiled CALL whose site had settled on a builtin when the code compiled
now passes the helper a per-site record (the builtin, which the native code
holds, its leaf kind and kernel, the operand count). The helper checks the
callee against the builtin by address instead of looking up the site's
method slot and leaf kind, runs a container's kernel or the leaf body in
place, and releases the operands with an in-line plain decrement (a last
owner, or a watched release, takes Interpreter::release). A site settled on
a builtin body the core loop's lane runs gets a helper that goes straight
to that body.

The lane's body (core_native_body, shared by the core loop and the JIT)
now runs on the operands where they sit on the stack, as a vectorcall
reads them, instead of copying them into a buffer first; they leave the
stack and are released, arguments first, right after the call. A fused
x.m(...) on a local no longer tries h_local_method for a call site settled
on a non-leaf body, which only leaf calls take.

Instructions per iteration (probe, before -> after; CPython 3.14):
d.setdefault + d.pop 3040 -> 2000 (1244), two getattr(o, n, d) 2503 ->
1924 (1397), with Lock 2051 -> 1533 (1699), list subclass append + pop
3650 -> 2319 (1179), bytes hex + count 4699 -> 4047.

Suite geomean against CPython 1.419 -> 1.407 (0.991x; context_managers
0.965x, deepcopy 0.975x, text_processing 0.974x, graph_search 0.976x).
---
 crates/weavepy-vm/src/frame_jit.rs            | 388 ++++++++++++------
 crates/weavepy-vm/src/lib.rs                  | 274 ++++++++-----
 .../regrtest/test_frame_jit_builtin_sites.py  | 282 +++++++++++++
 3 files changed, 718 insertions(+), 226 deletions(-)
 create mode 100644 tests/regrtest/test_frame_jit_builtin_sites.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index d9d896d7..d007fd82 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -209,6 +209,9 @@ pub(crate) struct Native {
     /// The `CALL` sites' direct-call caches.
     #[allow(clippy::vec_box)]
     _direct: Vec<Box<DirectSite>>,
+    /// The `CALL` sites' settled builtins.
+    #[allow(clippy::vec_box)]
+    _builtin_sites: Vec<Box<BuiltinSite>>,
     /// The builtins the method kernels' calls address.
     _held: Vec<Rc<crate::object::BuiltinFn>>,
     #[allow(clippy::vec_box)]
@@ -1795,36 +1798,6 @@ fn kernel_for(tag: u64, kind: crate::LeafKind, nargs: usize) -> Option<Kernel> {
     })
 }
 
-/// The call of method body `b` of leaf kind `kind` on `args` (the
-/// receiver first) through its [`Kernel`], for a `list`, `dict`, `set` or
-/// `str` receiver that has one: its result, or `None` (nothing done).
-/// (Out of line: the general call helper stays as it was.)
-#[inline(never)]
-fn kernel_call(
-    kind: crate::LeafKind,
-    args: &[Object],
-    b: &Rc<crate::object::BuiltinFn>,
-) -> Option<Object> {
-    let tag = match args.first()? {
-        Object::List(_) => 1,
-        Object::Dict(_) => 2,
-        Object::Set(_) => 3,
-        Object::Str(_) => 4,
-        _ => return None,
-    };
-    if args.len() > 3 {
-        return None;
-    }
-    let kernel = kernel_for(tag, kind, args.len() - 1)?;
-    let arg = |k: usize| args.get(k).map_or(std::ptr::null(), std::ptr::from_ref);
-    let mut out = std::mem::MaybeUninit::<Object>::uninit();
-    // SAFETY: the operands are initialized values; `out` is free.
-    unsafe {
-        (kernel(out.as_mut_ptr(), &args[0], arg(1), arg(2), Rc::as_ptr(b)) == 0)
-            .then(|| out.assume_init())
-    }
-}
-
 /// The method's own body on bitwise views of the operands (as the core
 /// loop's leaf call runs it): `0` its result written to `out`, anything
 /// else (an error, which the full call raises) declined.
@@ -5435,11 +5408,29 @@ fn len_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
     Some(Ok(Object::Int(n as i64)))
 }
 
-/// [`h_call`] at a site settled on a leaf builtin: a container's or
-/// string's method through its [`Kernel`], or `isinstance`, `len`, a
-/// registered fast half or a registered leaf body in place as `h_call`
-/// runs them, from a small function (the general helper stays as it
-/// is); anything else (another callee, a declined shape) through
+/// A `CALL` site settled on a builtin when its code compiled (see
+/// `Lower::call_op`): the builtin (the native code holds it, so its address
+/// names it for as long as the code lives), the operand count, and how the
+/// site runs it: a leaf kind (with the [`Kernel`] its receiver variant
+/// takes, if any, and that variant's tag), or a body through the core
+/// loop's lane.
+pub(crate) struct BuiltinSite {
+    f: *const crate::object::BuiltinFn,
+    argc: usize,
+    kind: Option<crate::LeafKind>,
+    /// The leaf kind runs in place for any receiver (see
+    /// `LeafKind::runs_in_core`).
+    in_core: bool,
+    kernel: Option<Kernel>,
+    recv_tag: u8,
+    via_call: bool,
+}
+
+/// [`h_call`] at a site settled on a leaf builtin `site` names: the
+/// callee checked to be that builtin by its address, then a container's or
+/// string's method through its [`Kernel`] on the receiver in the self slot,
+/// or the leaf body the site's kind names in place, the operands released
+/// after; anything else (another callee, a declined shape) through
 /// `h_call`.
 unsafe extern "C" fn h_call_leaf(
     st: *mut State,
@@ -5447,99 +5438,172 @@ unsafe extern "C" fn h_call_leaf(
     ext: *const CodeConstObjects,
     pc: u64,
     len: u64,
+    site: *const BuiltinSite,
 ) -> u32 {
     use crate::LeafKind;
-    // SAFETY: as `h_call`'s.
+    // SAFETY: as `h_call`'s; the site is this `CALL`'s, and the code holds
+    // its builtin.
     unsafe {
         let s = &mut *st;
-        let (c, x, pcu, lenu) = (&*code, &*ext, pc as usize, len as usize);
-        let argc = c.instructions[pcu].arg as usize;
-        if lenu >= argc + 2 && !s.sw.is_null() {
-            let base = s.stack;
-            let start = lenu - argc - 2;
-            if let Object::Builtin(b) = &*base.add(start) {
-                let kind = x.method_slot(pcu).and_then(|m| m.get_leaf(b));
+        let site = &*site;
+        let lenu = len as usize;
+        let start = lenu - site.argc - 2;
+        let base = s.stack;
+        let hit = matches!(&*base.add(start), Object::Builtin(b) if std::ptr::eq(Rc::as_ptr(b), site.f));
+        if hit && !s.sw.is_null() {
+            let recv = &*base.add(start + 1);
+            let r = match site.kernel {
                 // A container's or string's method with a kernel, on the
                 // receiver in the self slot.
-                if let (Some(k), false) = (kind, matches!(&*base.add(start + 1), Object::Unbound)) {
-                    let args = std::slice::from_raw_parts(base.add(start + 1), argc + 1);
-                    if Rc::strong_count(b) > 1 && Interpreter::core_all_droppable(args) {
-                        if let Some(v) = kernel_call(k, args, b) {
-                            crate::release_run(base.add(start), lenu - start);
-                            base.add(start).write(v);
-                            s.len = start + 1;
-                            return 0;
+                Some(kernel) if recv.tag() == site.recv_tag => {
+                    let arg = |k: usize| {
+                        if k <= site.argc {
+                            base.add(start + 1 + k).cast_const()
+                        } else {
+                            std::ptr::null()
                         }
-                    }
+                    };
+                    let mut out = std::mem::MaybeUninit::<Object>::uninit();
+                    (kernel(out.as_mut_ptr(), recv, arg(1), arg(2), site.f) == 0)
+                        .then(|| Ok(out.assume_init()))
                 }
-                if matches!(
-                    kind,
-                    Some(
-                        LeafKind::Isinstance
-                            | LeafKind::Len
-                            | LeafKind::Fast(_)
-                            | LeafKind::Opaque
-                            | LeafKind::Scalar
-                    )
-                ) && Rc::strong_count(b) > 1
-                {
-                    let first = if matches!(&*base.add(start + 1), Object::Unbound) {
+                // (A kernel's kind runs nothing in place on another
+                // receiver: through `h_call`.)
+                Some(_) => None,
+                None if site.in_core => {
+                    let first = if matches!(recv, Object::Unbound) {
                         start + 2
                     } else {
                         start + 1
                     };
                     let args = std::slice::from_raw_parts(base.add(first), lenu - first);
-                    let r = if Interpreter::core_all_droppable(args) {
-                        // (A body that runs no Python code for any
-                        // arguments, or for scalar ones: as `h_call` and
-                        // `leaf_builtin_call` run them.)
-                        let body = || {
-                            Some(match b.call_kw.as_ref() {
-                                Some(ckw) => ckw(args, &[]),
-                                None => (b.call)(args),
-                            })
-                        };
-                        match kind {
-                            Some(LeafKind::Fast(f)) => f(args),
-                            Some(LeafKind::Len) => len_fast(args),
-                            Some(LeafKind::Opaque) => body(),
-                            Some(LeafKind::Scalar) => {
-                                if args.iter().all(|a| {
-                                    matches!(a, Object::Int(_) | Object::Float(_) | Object::Bool(_))
-                                }) {
-                                    body()
-                                } else {
-                                    None
-                                }
-                            }
-                            _ => Interpreter::core_isinstance(args),
-                        }
-                    } else {
-                        None
+                    let b = &*site.f;
+                    // (A body that runs no Python code for any arguments,
+                    // or for scalar ones: as `h_call` and
+                    // `leaf_builtin_call` run them.)
+                    let body = || {
+                        Some(match b.call_kw.as_ref() {
+                            Some(ckw) => ckw(args, &[]),
+                            None => (b.call)(args),
+                        })
                     };
-                    if let Some(r) = r {
-                        crate::release_run(base.add(start), lenu - start);
-                        return match r {
-                            Ok(v) => {
-                                base.add(start).write(v);
-                                s.len = start + 1;
-                                0
-                            }
-                            Err(e) => {
-                                s.len = start;
-                                s.pc = pcu + 1;
-                                s.err = Some(e);
-                                RAISED
+                    match site.kind {
+                        Some(LeafKind::Fast(f)) => f(args),
+                        Some(LeafKind::Len) => len_fast(args),
+                        Some(LeafKind::Isinstance) => Interpreter::core_isinstance(args),
+                        Some(LeafKind::Opaque) => body(),
+                        Some(LeafKind::Scalar) => {
+                            if args.iter().all(|a| {
+                                matches!(a, Object::Int(_) | Object::Float(_) | Object::Bool(_))
+                            }) {
+                                body()
+                            } else {
+                                None
                             }
-                        };
+                        }
+                        Some(k) => (*s.interp).leaf_builtin_call(k, b, args),
+                        None => None,
                     }
                 }
+                None => None,
+            };
+            if let Some(r) = r {
+                // The operands leave by their releases.
+                crate::release_operands(&*s.interp, base.add(start), lenu - start);
+                return match r {
+                    Ok(v) => {
+                        base.add(start).write(v);
+                        s.len = start + 1;
+                        0
+                    }
+                    Err(e) => {
+                        s.len = start;
+                        s.pc = pc as usize + 1;
+                        s.err = Some(e);
+                        RAISED
+                    }
+                };
             }
         }
         h_call(st, code, ext, pc, len)
     }
 }
 
+/// [`h_call`] at a site settled on a builtin whose body the core loop's
+/// lane runs (see `Interpreter::core_native_body`), named by `site`: the
+/// callee checked to be that builtin by its address (a method with its
+/// receiver in the self slot, or a function with the slot empty), the
+/// frame synced, and the body run on the operands in place. `0` the result
+/// in the callee's slot (the same activation running, nothing pending),
+/// `RAISED` the body raised (likewise), `RELOAD` the core loop goes on
+/// from the synced frame; anything else (another callee or shape) through
+/// `h_call`.
+unsafe extern "C" fn h_call_native(
+    st: *mut State,
+    code: *const CodeObject,
+    ext: *const CodeConstObjects,
+    pc: u64,
+    len: u64,
+    site: *const BuiltinSite,
+) -> u32 {
+    // SAFETY: as `h_call`'s; the site is this `CALL`'s, and the code holds
+    // its builtin.
+    unsafe {
+        let s = &mut *st;
+        let site = &*site;
+        let (pcu, lenu) = (pc as usize, len as usize);
+        let start = lenu - site.argc - 2;
+        let base = s.stack;
+        let hit = match (&*base.add(start), &*base.add(start + 1)) {
+            (Object::Builtin(b), Object::Unbound) => {
+                std::ptr::eq(Rc::as_ptr(b), site.f) && !b.binds_instance
+            }
+            (Object::Builtin(b), recv) => {
+                std::ptr::eq(Rc::as_ptr(b), site.f) && crate::lane_recv_ok(b, recv, site.via_call)
+            }
+            _ => false,
+        };
+        if !hit || s.sw.is_null() {
+            return h_call(st, code, ext, pc, len);
+        }
+        // The frame synced, as the core loop's arm syncs it.
+        let frame = &mut *s.frame;
+        frame.stack.set_len(lenu);
+        let sw = &mut *s.sw;
+        *sw.last = s.last;
+        let interp = &mut *s.interp.cast_mut();
+        let r = interp.core_native_body(sw, frame, pcu, start, site.f, site.via_call);
+        let in_place = sw.pending.is_none()
+            && std::ptr::eq(sw.cur, s.frame)
+            && crate::hot_gates::loop_gen() == s.snap_gen;
+        match r {
+            Ok(v) => {
+                base.add(start).write(v);
+                frame.stack.set_len(start + 1);
+                if in_place {
+                    s.len = start + 1;
+                    return 0;
+                }
+                *sw.last = pcu;
+                if sw.pending.is_none() && (*sw.maybe_dead).get() {
+                    sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Marked));
+                }
+                RELOAD
+            }
+            Err(e) if in_place => {
+                s.len = start;
+                s.pc = pcu + 1;
+                s.err = Some(e);
+                RAISED
+            }
+            Err(e) => {
+                sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Raised(e)));
+                RELOAD
+            }
+        }
+    }
+}
+
 /// The `CALL` at `pc` (its operands from `start` up a `len`-deep stack) of a
 /// leaf builtin whose leaf half declined, or of an instance's
 /// `__reduce_ex__`, through the core loop's lane for them
@@ -6567,6 +6631,7 @@ fn compile_with(
     let mut slots: Vec<Box<SlotCache>>;
     let mut methods: Vec<Box<MethodCache>>;
     let mut direct: Vec<Box<DirectSite>>;
+    let mut builtin_sites: Vec<Box<BuiltinSite>>;
     let mut held: Vec<Rc<crate::object::BuiltinFn>>;
     let mut mods: Vec<Box<ModCache>>;
     let t0 = stats::enabled().then(std::time::Instant::now);
@@ -6584,6 +6649,7 @@ fn compile_with(
         slots = std::mem::take(&mut lower.slot_caches);
         methods = std::mem::take(&mut lower.method_caches);
         direct = std::mem::take(&mut lower.direct_sites);
+        builtin_sites = std::mem::take(&mut lower.builtin_sites);
         held = std::mem::take(&mut lower.held);
         mods = std::mem::take(&mut lower.mod_caches);
         // (Measured as if every exit's tail were in line, the size the
@@ -6654,6 +6720,7 @@ fn compile_with(
         _slots: slots,
         _methods: methods,
         _direct: direct,
+        _builtin_sites: builtin_sites,
         _held: held,
         _mods: mods,
     })
@@ -6766,6 +6833,9 @@ struct Lower<'a> {
     method_caches: Vec<Box<MethodCache>>,
     #[allow(clippy::vec_box)]
     direct_sites: Vec<Box<DirectSite>>,
+    /// The `CALL` sites' settled builtins (see [`BuiltinSite`]).
+    #[allow(clippy::vec_box)]
+    builtin_sites: Vec<Box<BuiltinSite>>,
     /// The builtins the method kernels' calls address.
     held: Vec<Rc<crate::object::BuiltinFn>>,
     /// The method loads' module-attribute caches.
@@ -6948,6 +7018,7 @@ impl<'a> Lower<'a> {
             slot_caches: Vec::new(),
             method_caches: Vec::new(),
             direct_sites: Vec::new(),
+            builtin_sites: Vec::new(),
             held: Vec::new(),
             mod_caches: Vec::new(),
             catch: None,
@@ -8338,7 +8409,9 @@ impl<'a> Lower<'a> {
         // here and leaves at the `CALL`.
         match self.code.instructions.get(pc + 1).map(|i| i.op) {
             Some(OpCode::LoadMethodAttr)
-                if !self.is_gen() && crate::method_site_in_place(self.ext, pc + 1) =>
+                if !self.is_gen()
+                    && crate::method_site_in_place(self.ext, pc + 1)
+                    && !self.native_body_call(pc + 2) =>
             {
                 self.method_kernel(pc);
                 self.method_call(pc)
@@ -8364,6 +8437,30 @@ impl<'a> Lower<'a> {
         true
     }
 
+    /// Whether the simple argument loads from `pc` end in a `CALL` whose
+    /// site settled on a builtin body the core loop's lane runs: the fused
+    /// method call (`h_local_method`) runs leaf calls only, so the method
+    /// loads and its call runs through [`h_call_native`].
+    fn native_body_call(&self, pc: usize) -> bool {
+        let ins = &self.code.instructions;
+        let mut n = 0;
+        for k in pc..ins.len().min(pc + 16) {
+            match ins[k].op {
+                OpCode::LoadFast
+                | OpCode::LoadFastBorrow
+                | OpCode::LoadConst
+                | OpCode::LoadSmallInt
+                | OpCode::BinaryOp => n += 1,
+                OpCode::Call => {
+                    return n >= ins[k].arg as usize
+                        && matches!(crate::call_site_builtin(self.ext, k), Some((_, None, _)));
+                }
+                _ => return false,
+            }
+        }
+        false
+    }
+
     /// Whether the code is a generator's, coroutine's or async generator's
     /// body: its resume can't raise from here (its calls are the general
     /// loop's).
@@ -10556,6 +10653,16 @@ impl<'a> Lower<'a> {
         let ext = self.ext_ptr();
         let pcv = self.b.ins().iconst(types::I64, pc as i64);
         let len = self.b.ins().iconst(types::I64, self.depth as i64);
+        // A site settled on a builtin calls it through its own helper,
+        // which checks the callee against the site's builtin by address
+        // (anything else goes on to `h_call`).
+        if let Some(helper) = self.builtin_site(pc, argc) {
+            let (helper, site) = helper;
+            let r = self
+                .call(helper, &[self.st, code, ext, pcv, len, site], true)
+                .expect("returns");
+            return self.called(pc, argc, r);
+        }
         // A Python function's call tries a direct call first, unless its
         // site found the callee without a native body and has retries
         // left to count down.
@@ -10604,20 +10711,71 @@ impl<'a> Lower<'a> {
             .expect("returns");
         self.b.ins().jump(called, &[r.into()]);
         self.b.switch_to_block(general);
-        // (A site settled on a leaf builtin tries it first, in a smaller
-        // helper.)
-        let leaf = crate::call_site_leaf(self.ext, pc).is_some();
-        let helper = if leaf {
-            h_call_leaf as *const () as usize
-        } else {
-            h_call as *const () as usize
-        };
         let r = self
-            .call(helper, &[self.st, code, ext, pcv, len], true)
+            .call(
+                h_call as *const () as usize,
+                &[self.st, code, ext, pcv, len],
+                true,
+            )
             .expect("returns");
         self.b.ins().jump(called, &[r.into()]);
         self.b.switch_to_block(called);
         let r = self.b.block_params(called)[0];
+        self.called(pc, argc, r)
+    }
+
+    /// The helper and its site for a `CALL` (of `argc` arguments) at `pc`
+    /// whose site settled on a builtin (see [`BuiltinSite`]): a leaf kind
+    /// with a kernel or one run in place through [`h_call_leaf`], a body
+    /// through the core loop's lane through [`h_call_native`]; `None` for
+    /// anything else (`h_call` alone).
+    fn builtin_site(&mut self, pc: usize, argc: usize) -> Option<(usize, Value)> {
+        use crate::LeafKind;
+        let (f, kind, via_call) = crate::call_site_builtin(self.ext, pc)?;
+        let (kernel, recv_tag) = match kind {
+            // (An opaque body's kernel is the body: run in place on any
+            // receiver below.)
+            Some(k) if k != LeafKind::Opaque => (1..=4)
+                .find_map(|t| kernel_for(t, k, argc).map(|kn| (Some(kn), t)))
+                .map_or((None, 0), |(kn, t)| {
+                    let tag = match t {
+                        1 => self.tags.list,
+                        2 => self.tags.dict,
+                        3 => self.tags.set,
+                        _ => self.tags.str,
+                    };
+                    (kn, tag)
+                }),
+            _ => (None, 0),
+        };
+        let in_core = kind.is_some_and(LeafKind::runs_in_core);
+        let helper = match kind {
+            None => h_call_native as *const () as usize,
+            Some(_) if kernel.is_some() || in_core => h_call_leaf as *const () as usize,
+            Some(_) => return None,
+        };
+        let site = Box::new(BuiltinSite {
+            f: Rc::as_ptr(&f),
+            argc,
+            kind,
+            in_core,
+            kernel,
+            recv_tag,
+            via_call,
+        });
+        self.held.push(f);
+        let at = self
+            .b
+            .ins()
+            .iconst(self.ptr, std::ptr::from_ref::<BuiltinSite>(&site) as i64);
+        self.builtin_sites.push(site);
+        Some((helper, at))
+    }
+
+    /// After a `CALL`'s helper returned `r`: the result in place on `0`
+    /// (the callee and `argc` operands gone), the core loop's on anything
+    /// else.
+    fn called(&mut self, pc: usize, argc: usize, r: Value) -> bool {
         let out = self.exit_with(pc, &[], INTERP);
         let ran = self.b.create_block();
         let other = self.b.create_block();
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index fd59a449..0ac98e4d 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -15159,129 +15159,38 @@ impl Interpreter {
     /// The core loop's `CALL` at `pc` of a non-leaf builtin the site settled
     /// on (see [`Self::builtin_lane`]), in `sw`'s synced running activation:
     /// a builtin type's method on its exact receiver, as `(method, receiver)`
-    /// or as a bound method with an empty self slot, or a builtin function.
-    /// The body runs directly, or with `via_call` through [`Self::call`], as
-    /// the full handler's dispatch ends up running it, without that
-    /// handler's chain of special cases. The body may run Python code (an
-    /// element's `__eq__`, a key function), so the operands leave the stack
-    /// first, the activation waits on the pending list meanwhile, and the
-    /// core loop reloads after. `false` touches nothing.
+    /// or as a bound method with an empty self slot, or a builtin function,
+    /// run by [`Self::core_native_body`]. The core loop reloads after.
+    /// `false` touches nothing.
     #[inline(never)]
     fn core_builtin_lane(&mut self, sw: &mut CoreSwitch, pc: usize, via_call: bool) -> bool {
-        const MAX_ARGS: usize = 8;
         // SAFETY: see `CoreSwitch`: the running activation is synced and
         // unborrowed here.
         let frame = unsafe { &mut *sw.cur };
         let Some(ins) = frame.code.instructions.get(pc) else {
             return false;
         };
-        let argc = ins.arg as usize;
-        let n = frame.stack.len();
-        let Some(callee_at) = n.checked_sub(argc + 2) else {
+        let Some(callee_at) = frame.stack.len().checked_sub(ins.arg as usize + 2) else {
             return false;
         };
-        if argc >= MAX_ARGS {
-            return false;
-        }
-        let recv_ok = |b: &crate::object::BuiltinFn, recv: &Object| {
-            b.binds_instance
-                && (!via_call
-                    || builtins::method_memo_tag(recv).is_some()
-                    || exception_init_call(b, recv)
-                    || (matches!(b.name, ".object_reduce_ex" | ".object_reduce")
-                        && matches!(recv, Object::Instance(_))))
-        };
-        let ok = match (&frame.stack[callee_at], &frame.stack[callee_at + 1]) {
-            (Object::Builtin(b), Object::Unbound) => !b.binds_instance,
-            (Object::Builtin(b), recv) => recv_ok(b, recv),
-            (Object::BoundMethod(bm), Object::Unbound) if !bm.redispatch_descriptor => {
-                match &bm.function {
-                    Object::Builtin(b) => recv_ok(b, &bm.receiver),
-                    _ => false,
+        let b: *const crate::object::BuiltinFn =
+            match (&frame.stack[callee_at], &frame.stack[callee_at + 1]) {
+                (Object::Builtin(b), Object::Unbound) if !b.binds_instance => Rc::as_ptr(b),
+                (Object::Builtin(b), recv) if lane_recv_ok(b, recv, via_call) => Rc::as_ptr(b),
+                (Object::BoundMethod(bm), Object::Unbound) if !bm.redispatch_descriptor => {
+                    match &bm.function {
+                        Object::Builtin(b) if lane_recv_ok(b, &bm.receiver, via_call) => {
+                            Rc::as_ptr(b)
+                        }
+                        _ => return false,
+                    }
                 }
-            }
-            _ => false,
-        };
-        if !ok {
-            return false;
-        }
-        // Committed. The callable and the self slot leave the stack, the
-        // receiver (the self slot's, moved, or a bound method's) and the
-        // arguments into `buf`.
-        let mut buf = [const { std::mem::MaybeUninit::<Object>::uninit() }; MAX_ARGS + 1];
-        // SAFETY: the callable, the self slot and the `argc` operands
-        // above it leave the stack, which forgets them (`argc < MAX_ARGS`).
-        let (callable, moved_self) = unsafe {
-            let base = frame.stack.as_ptr().add(callee_at);
-            let callable = base.read();
-            let self_slot = base.add(1).read();
-            let moved_self = !matches!(self_slot, Object::Unbound);
-            if moved_self {
-                buf[0].write(self_slot);
-            } else if let Object::BoundMethod(bm) = &callable {
-                buf[0].write(bm.receiver.clone());
-            }
-            let first = usize::from(moved_self || matches!(callable, Object::BoundMethod(_)));
-            for k in 0..argc {
-                buf[first + k].write(base.add(2 + k).read());
-            }
-            frame.stack.set_len(callee_at);
-            (callable, moved_self)
-        };
-        let b: &crate::object::BuiltinFn = match &callable {
-            Object::Builtin(b) => b,
-            Object::BoundMethod(bm) => match &bm.function {
-                Object::Builtin(b) => b,
-                _ => unreachable!("checked above"),
-            },
-            _ => unreachable!("checked above"),
-        };
-        let first = usize::from(moved_self || matches!(callable, Object::BoundMethod(_)));
-        frame.pc = pc as u32 + 1;
-        // SAFETY: the first `first + argc` entries were written above.
-        let ops = unsafe {
-            std::slice::from_raw_parts_mut(buf.as_mut_ptr().cast::<Object>(), first + argc)
-        };
-        // The body may run Python code that looks up the stack.
-        let pending = self.core_pending_enter_fast(sw, frame, pc);
-        let result = if via_call {
-            let globals = frame.globals.clone();
-            match &callable {
-                Object::Builtin(_) => self.call(&callable, ops, &[], &globals),
-                _ => self.call(&bm_builtin(&callable), ops, &[], &globals),
-            }
-        } else {
-            match b.call_kw.as_ref() {
-                Some(call_kw) => call_kw(ops, &[]),
-                None => (b.call)(ops),
-            }
-        };
-        self.core_pending_exit(sw, pending);
-        // The operands drop as the full handler's do, after the call: the
-        // arguments plainly, the receiver the self slot held and the
-        // callable as the full handler releases them (a scalar or a shared
-        // builtin by a plain drop).
-        // SAFETY: each initialized entry drops exactly once; `buf` itself
-        // has no drop glue.
-        unsafe {
-            if moved_self {
-                let receiver = ops.as_ptr().read();
-                std::ptr::drop_in_place(&mut ops[1..]);
-                self.release(receiver);
-            } else {
-                std::ptr::drop_in_place(ops);
-            }
-        }
-        if matches!(callable, Object::Builtin(_)) {
-            drop(callable);
-        } else {
-            self.release(callable);
-        }
-        match result {
+                _ => return false,
+            };
+        // SAFETY: the activation is synced, and the shape was checked.
+        match unsafe { self.core_native_body(sw, frame, pc, callee_at, b, via_call) } {
             Ok(v) => {
-                // SAFETY: as above (the body ran nested activations of its
-                // own, never this one).
-                unsafe { (*sw.cur).stack.push(v) };
+                frame.stack.push(v);
                 // SAFETY: the running activation's last-pc slot.
                 unsafe { *sw.last = pc };
             }
@@ -15296,6 +15205,80 @@ impl Interpreter {
         true
     }
 
+    /// The body of the non-leaf builtin `b` for the `CALL` at `pc` of `sw`'s
+    /// running activation `frame` (synced, its operands from `start` to the
+    /// top of its stack): `b` is the callee itself (a method with its
+    /// receiver in the self slot, or a function with the slot empty) or a
+    /// bound method's function, whose receiver fills the empty self slot
+    /// here. The body runs directly, or with `via_call` through
+    /// [`Self::call`], as the full handler's dispatch ends up running it,
+    /// on the operands where they are (as a vectorcall reads them, the
+    /// stack holding them), with the activation waiting on the pending
+    /// list: the body may run Python code (an element's `__eq__`, a key
+    /// function). Then the operands leave as the full handler's do, the
+    /// arguments first, then the receiver and the callable, and the stack
+    /// ends at `start` (the result is the caller's to place).
+    ///
+    /// # Safety
+    ///
+    /// `frame` is `sw`'s synced running activation, unborrowed elsewhere;
+    /// the callee at `start` is `b` (or a bound method of it) in a shape
+    /// [`Self::core_builtin_lane`] admits.
+    #[inline(never)]
+    pub(crate) unsafe fn core_native_body(
+        &mut self,
+        sw: &mut CoreSwitch,
+        frame: &mut Frame,
+        pc: usize,
+        start: usize,
+        b: *const crate::object::BuiltinFn,
+        via_call: bool,
+    ) -> Result<Object, RuntimeError> {
+        let len = frame.stack.len();
+        // SAFETY: the caller's contract: `start + 2 <= len`; the stack
+        // isn't resized while the body runs (nested code runs activations
+        // of its own).
+        unsafe {
+            let base = frame.stack.as_mut_ptr();
+            if matches!(&*base.add(start + 1), Object::Unbound) {
+                if let Object::BoundMethod(bm) = &*base.add(start) {
+                    // (The empty self slot needs no release.)
+                    base.add(start + 1).write(bm.receiver.clone());
+                }
+            }
+            let first = if matches!(&*base.add(start + 1), Object::Unbound) {
+                start + 2
+            } else {
+                start + 1
+            };
+            frame.pc = pc as u32 + 1;
+            // The body may run Python code that looks up the stack.
+            let pending = self.core_pending_enter_fast(sw, frame, pc);
+            let args = std::slice::from_raw_parts(base.add(first), len - first);
+            let result = if via_call {
+                let globals = frame.globals.clone();
+                match &*base.add(start) {
+                    callable @ Object::Builtin(_) => self.call(callable, args, &[], &globals),
+                    callable => self.call(&bm_builtin(callable), args, &[], &globals),
+                }
+            } else {
+                let b = &*b;
+                match b.call_kw.as_ref() {
+                    Some(call_kw) => call_kw(args, &[]),
+                    None => (b.call)(args),
+                }
+            };
+            self.core_pending_exit(sw, pending);
+            // The operands leave the stack, then are released: the
+            // arguments, the receiver, the callable.
+            frame.stack.set_len(start);
+            release_operands(self, base.add(start + 2), len - start - 2);
+            release_operands(self, base.add(start + 1), 1);
+            release_operands(self, base.add(start), 1);
+            result
+        }
+    }
+
     /// A `BINARY_OP` or `COMPARE_OP` at `sw`'s running activation's `pc`
     /// with an instance operand (a Python dunder, `Vector.__sub__`): the
     /// full handler's own step, run with the activation on the pending
@@ -72268,6 +72251,20 @@ fn exception_init_call(b: &crate::object::BuiltinFn, recv: &Object) -> bool {
     b.name == "__init__" && matches!(recv, Object::Instance(i) if i.cls_raw().flags.is_exception)
 }
 
+/// Whether the core loop's builtin lane may call the method body `b` on
+/// `recv` (see `Interpreter::core_builtin_lane`): a body that binds
+/// instances, and with `via_call` one whose receiver the interpreter's
+/// dispatch knows by name.
+#[inline]
+fn lane_recv_ok(b: &crate::object::BuiltinFn, recv: &Object, via_call: bool) -> bool {
+    b.binds_instance
+        && (!via_call
+            || builtins::method_memo_tag(recv).is_some()
+            || exception_init_call(b, recv)
+            || (matches!(b.name, ".object_reduce_ex" | ".object_reduce")
+                && matches!(recv, Object::Instance(_))))
+}
+
 /// The builtin function a bound method of one wraps, as its own callable.
 #[cold]
 fn bm_builtin(callable: &Object) -> Object {
@@ -73382,6 +73379,24 @@ pub(crate) fn call_site_leaf(ext: &CodeConstObjects, pc: usize) -> Option<LeafKi
     }
 }
 
+/// The builtin the `CALL` at `pc` last settled on: its leaf kind, or
+/// `None` and whether its body runs through `Interpreter::call` for a
+/// builtin whose body the core loop's lane runs (see
+/// [`MethodSlotFn::NativeBody`]).
+#[cfg(feature = "jit")]
+pub(crate) fn call_site_builtin(
+    ext: &CodeConstObjects,
+    pc: usize,
+) -> Option<(Rc<crate::object::BuiltinFn>, Option<LeafKind>, bool)> {
+    let slot = ext.method_slot(pc)?;
+    // SAFETY: GIL-serialized; the borrow ends here.
+    match unsafe { &(*slot.0.get()).1 } {
+        MethodSlotFn::Leaf(f, kind) => Some((f.clone(), Some(*kind), false)),
+        MethodSlotFn::NativeBody(f, via_call) => Some((f.clone(), None, *via_call)),
+        _ => None,
+    }
+}
+
 /// Whether the `CALL` at `pc` last settled on a leaf builtin the core
 /// loop's arm (and the frame JIT's helper) runs in place.
 #[cfg(feature = "jit")]
@@ -73707,6 +73722,43 @@ fn drop_hot(v: Object) {
     }
 }
 
+/// [`Interpreter::release`] of a call's operand with its commonest case in
+/// line: a scalar, or a shared value's plain decrement while nothing
+/// watches releases (one load of [`plain_release_gate`]'s flags). A last
+/// owner, or any release an observer wants, takes the full release.
+#[inline(always)]
+fn release_hot(interp: &Interpreter, v: Object) {
+    if let Some(word) = v.strong_word() {
+        // SAFETY: the gates are four flags laid out as one `u32`; `v` owns
+        // a reference to the live allocation, released here unless it's
+        // the last one.
+        let plain = unsafe {
+            (*plain_release_gate().cast::<std::sync::atomic::AtomicU32>())
+                .load(std::sync::atomic::Ordering::Relaxed)
+                == 0
+                && crate::rc::try_release_word(word)
+        };
+        if !plain {
+            return interp.release(v);
+        }
+    }
+    std::mem::forget(v);
+}
+
+/// Release the `n` call operands at `p` (moved out: the caller's stack no
+/// longer counts them) in order, as [`release_hot`] does.
+///
+/// # Safety
+///
+/// `p` holds `n` initialized values the caller owns and gives up.
+#[inline(always)]
+unsafe fn release_operands(interp: &Interpreter, p: *const Object, n: usize) {
+    for k in 0..n {
+        // SAFETY: the caller's contract.
+        release_hot(interp, unsafe { p.add(k).read() });
+    }
+}
+
 /// The stamp slot for `cache_pc`, allocating the table on first use.
 #[inline]
 fn code_stamp_slot(code: &CodeObject, cache_pc: u32) -> Option<&StampSlot> {
diff --git a/tests/regrtest/test_frame_jit_builtin_sites.py b/tests/regrtest/test_frame_jit_builtin_sites.py
new file mode 100644
index 00000000..c7ef275b
--- /dev/null
+++ b/tests/regrtest/test_frame_jit_builtin_sites.py
@@ -0,0 +1,282 @@
+"""Builtin calls from compiled code at sites settled on one builtin.
+
+A compiled `CALL` whose site settled on a builtin checks the callee against
+that builtin by address and runs its leaf half or its body on the operands
+in place. A different callee at the same site (a rebound or shadowed
+name, another builtin, a Python function) must take the general path; a
+body that raises must leave the right traceback; operands must be released
+right after the call (`__del__` timing); a body that runs Python code must
+see the caller's frame whole and keep the operands alive even when that
+code rebinds the caller's locals; and `sys.setprofile` installed from a
+callback must see the calls after it.
+"""
+
+import builtins
+import os
+import subprocess
+import sys
+import unittest
+
+FORCED = "--forced-frame-jit"
+N = 3000
+
+
+def lengths(xs, n):
+    t = 0
+    for i in range(n):
+        t += len(xs)
+    return t
+
+
+def rebind_len(n):
+    global len
+    t = 0
+    try:
+        for i in range(n):
+            if i == n // 2:
+                len = lambda x: 100  # noqa: E731
+            t += len("ab")
+    finally:
+        del len
+    return t
+
+
+def mixed_callees(n):
+    fs = [len, abs, str.upper, lambda x: 7]
+    t = []
+    for i in range(n):
+        f = fs[(i // 50) % len(fs)]
+        try:
+            t.append(f("ab"))
+        except TypeError:
+            t.append(None)
+    return t
+
+
+def setdefaults(d, n):
+    t = 0
+    for i in range(n):
+        t += d.setdefault("a", 2)
+        t += d.pop("z", 0)
+    return t
+
+
+def pops(d, keys):
+    out = []
+    for k in keys:
+        try:
+            out.append(d.pop(k))
+        except KeyError as e:
+            out.append(("missing", e.args[0], e.__traceback__.tb_lineno))
+    return out
+
+
+POP_LINE = pops.__code__.co_firstlineno + 4
+
+
+def raising_pop(d, n):
+    for i in range(n):
+        d.pop(i)
+
+
+class Noisy:
+    log = []
+
+    def __init__(self, tag):
+        self.tag = tag
+
+    def __del__(self):
+        Noisy.log.append(("del", self.tag))
+
+
+def del_timing(n):
+    d = {"k": None}
+    for i in range(n):
+        d.setdefault("k", Noisy(i))
+        Noisy.log.append(("after", i))
+
+
+class FrameKey:
+    """A key whose `__eq__` looks at its caller's frame."""
+
+    seen = []
+
+    def __init__(self, v):
+        self.v = v
+
+    def __hash__(self):
+        return 1
+
+    def __eq__(self, other):
+        f = sys._getframe(1)
+        while f is not None and f.f_code.co_name != "frame_lookups":
+            f = f.f_back
+        FrameKey.seen.append(f.f_lineno if f is not None else None)
+        return isinstance(other, FrameKey) and self.v == other.v
+
+
+def frame_lookups(d, n):
+    t = 0
+    for i in range(n):
+        t += d.setdefault(FrameKey(1), 0)
+    return t
+
+
+LOOKUP_LINE = frame_lookups.__code__.co_firstlineno + 3
+
+
+class Rebinder:
+    """A key whose `__eq__` drops the caller's own reference to the dict."""
+
+    def __hash__(self):
+        return 2
+
+    def __eq__(self, other):
+        f = sys._getframe(1)
+        while f is not None and f.f_code.co_name != "rebinding":
+            f = f.f_back
+        if f is not None:
+            f.f_locals["d"] = None
+        return True
+
+
+def rebinding(n):
+    out = 0
+    for i in range(n):
+        d = {Rebinder(): i}
+        out += d.setdefault(Rebinder(), -1)
+    return out
+
+
+class MyList(list):
+    def append(self, x):
+        super().append(x * 2)
+
+
+class PlainList(list):
+    pass
+
+
+def appends(lst, n):
+    for i in range(n):
+        lst.append(i)
+        lst.count(i)
+    return lst
+
+
+def kw_calls(n):
+    t = 0
+    for i in range(n):
+        t += int("11", base=2) + len(sorted([3, 1, 2], reverse=True))
+    return t
+
+
+def profiled(n, events):
+    t = 0
+    for i in range(n):
+        if i == n // 2:
+            sys.setprofile(lambda frame, event, arg: events.append((event, getattr(arg, "__name__", None))))
+        t += len("abc")
+        t += "a,b".count(",")
+    sys.setprofile(None)
+    return t
+
+
+class BuiltinSiteTest(unittest.TestCase):
+    def test_leaf_and_rebinding(self):
+        self.assertEqual(lengths([1, 2], N), 2 * N)
+        self.assertEqual(rebind_len(N), (N // 2) * 2 + (N - N // 2) * 100)
+        self.assertEqual(rebind_len(N), (N // 2) * 2 + (N - N // 2) * 100)
+
+    def test_patched_builtins(self):
+        saved = builtins.len
+        try:
+            self.assertEqual(lengths("abc", N), 3 * N)
+            builtins.len = lambda x: 1
+            self.assertEqual(lengths("abc", N), N)
+        finally:
+            builtins.len = saved
+        self.assertEqual(lengths("abc", N), 3 * N)
+
+    def test_mixed_callees(self):
+        out = mixed_callees(N)
+        for i, v in enumerate(out):
+            want = [2, None, "AB", 7][(i // 50) % 4]
+            self.assertEqual(v, want, i)
+
+    def test_native_body(self):
+        self.assertEqual(setdefaults({"a": 1}, N), N)
+        d = {}
+        self.assertEqual(setdefaults(d, N), 2 * N)
+        self.assertEqual(d, {"a": 2})
+
+    def test_raise_traceback(self):
+        d = {1: "a", 3: "c"}
+        out = pops(d, [1, 2, 3, 4] * 1000)
+        self.assertEqual(out[:4], ["a", ("missing", 2, POP_LINE), "c", ("missing", 4, POP_LINE)])
+        try:
+            raising_pop({}, N)
+        except KeyError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("no KeyError")
+        while tb.tb_next is not None:
+            tb = tb.tb_next
+        self.assertEqual(tb.tb_frame.f_code.co_name, "raising_pop")
+        self.assertEqual(tb.tb_lineno, raising_pop.__code__.co_firstlineno + 2)
+
+    def test_del_timing(self):
+        Noisy.log.clear()
+        del_timing(N)
+        log = Noisy.log
+        # Each default dies right after its `setdefault`, before the next
+        # statement.
+        for i in range(N):
+            self.assertEqual(log[2 * i : 2 * i + 2], [("del", i), ("after", i)], i)
+
+    def test_callback_sees_frame(self):
+        FrameKey.seen.clear()
+        d = {FrameKey(1): 5}
+        self.assertEqual(frame_lookups(d, N), 5 * N)
+        self.assertEqual(set(FrameKey.seen), {LOOKUP_LINE})
+
+    def test_callback_rebinds_receiver(self):
+        self.assertEqual(rebinding(N), sum(range(N)))
+
+    def test_subclasses(self):
+        a = appends(MyList(), 500)
+        self.assertEqual(a, [2 * i for i in range(500)])
+        b = appends(PlainList(), 500)
+        self.assertEqual(b, list(range(500)))
+        self.assertIs(type(b), PlainList)
+
+    def test_keywords(self):
+        self.assertEqual(kw_calls(N), N * 6)
+
+    def test_profile_from_compiled_code(self):
+        events = []
+        self.assertEqual(profiled(N, events), N * 4)
+        calls = [name for ev, name in events if ev == "c_call"]
+        self.assertEqual(calls.count("len"), N - N // 2)
+        self.assertEqual(calls.count("count"), N - N // 2)
+
+    def test_forced_frame_jit(self):
+        if FORCED in sys.argv:
+            self.skipTest("already forced")
+        env = dict(os.environ)
+        env.update(
+            WEAVEPY_JIT="0",
+            WEAVEPY_FRAME_JIT_TUNE="8,1,3,0",
+            WEAVEPY_FRAME_JIT_HOT="50",
+        )
+        r = subprocess.run(
+            [sys.executable, __file__, FORCED],
+            env=env,
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(r.returncode, 0, r.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main(argv=[a for a in sys.argv if a != FORCED])

From 6115896dec4561dde80fe7ff97236bee3a0e8bc6 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 18:08:23 -0700
Subject: [PATCH 120/173] perf: give kernel and leaf builtin sites their own
 small helpers

A compiled CALL settled on a container method with a kernel now calls
h_call_kernel, and one settled on another in-place leaf kind calls
h_call_leaf: each checks only what its kind needs, so the kernel path no
longer carries the general leaf path's large result plumbing in its frame.
The operand release loop reads the release gates once per call and tests
each operand's count where it sits, moving a value out only for the full
release.

Instructions per iteration (probe, previous commit -> this one): two
getattr(o, n, d) 1924 -> 1805, isinstance 624 -> 583, min + abs 897 ->
834, d.setdefault + d.pop 2000 -> 1946.

Suite geomean against CPython 1.407 -> 1.405 (0.990x of the parent of the
previous commit; deepcopy 0.975x -> 0.963x, text_processing 0.974x ->
0.964x).
---
 crates/weavepy-vm/src/frame_jit.rs | 180 ++++++++++++++++-------------
 crates/weavepy-vm/src/lib.rs       |  47 +++++---
 2 files changed, 127 insertions(+), 100 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index d007fd82..4dc7a583 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -5418,20 +5418,61 @@ pub(crate) struct BuiltinSite {
     f: *const crate::object::BuiltinFn,
     argc: usize,
     kind: Option<crate::LeafKind>,
-    /// The leaf kind runs in place for any receiver (see
-    /// `LeafKind::runs_in_core`).
-    in_core: bool,
     kernel: Option<Kernel>,
     recv_tag: u8,
     via_call: bool,
 }
 
-/// [`h_call`] at a site settled on a leaf builtin `site` names: the
-/// callee checked to be that builtin by its address, then a container's or
-/// string's method through its [`Kernel`] on the receiver in the self slot,
-/// or the leaf body the site's kind names in place, the operands released
-/// after; anything else (another callee, a declined shape) through
-/// `h_call`.
+/// [`h_call`] at a site settled on a container's or string's leaf method
+/// with a [`Kernel`] (`site`'s): the callee checked to be that builtin by
+/// its address and the receiver in the self slot to be the kernel's
+/// variant, then the kernel run on the operands in place and the operands
+/// released; anything else (another callee or receiver, a declined call)
+/// through `h_call`.
+unsafe extern "C" fn h_call_kernel(
+    st: *mut State,
+    code: *const CodeObject,
+    ext: *const CodeConstObjects,
+    pc: u64,
+    len: u64,
+    site: *const BuiltinSite,
+) -> u32 {
+    // SAFETY: as `h_call`'s; the site is this `CALL`'s, and the code holds
+    // its builtin.
+    unsafe {
+        let s = &mut *st;
+        let site = &*site;
+        let lenu = len as usize;
+        let start = lenu - site.argc - 2;
+        let base = s.stack;
+        let recv = base.add(start + 1);
+        if let (Object::Builtin(b), Some(kernel), false) = (&*base.add(start), site.kernel, s.sw.is_null()) {
+            if std::ptr::eq(Rc::as_ptr(b), site.f) && (*recv).tag() == site.recv_tag {
+                let arg = |k: usize| {
+                    if k <= site.argc {
+                        recv.add(k).cast_const()
+                    } else {
+                        std::ptr::null()
+                    }
+                };
+                let mut out = std::mem::MaybeUninit::<Object>::uninit();
+                if kernel(out.as_mut_ptr(), recv, arg(1), arg(2), site.f) == 0 {
+                    crate::release_operands(&*s.interp, base.add(start), lenu - start);
+                    base.add(start).write(out.assume_init());
+                    s.len = start + 1;
+                    return 0;
+                }
+            }
+        }
+        h_call(st, code, ext, pc, len)
+    }
+}
+
+/// [`h_call`] at a site settled on a leaf builtin whose kind runs in place
+/// (see `LeafKind::runs_in_core`; `site`'s): the callee checked to be that
+/// builtin by its address, then the leaf body the kind names run on the
+/// operands in place, which are released after; anything else (another
+/// callee, a declined shape) through `h_call`.
 unsafe extern "C" fn h_call_leaf(
     st: *mut State,
     code: *const CodeObject,
@@ -5451,78 +5492,55 @@ unsafe extern "C" fn h_call_leaf(
         let base = s.stack;
         let hit = matches!(&*base.add(start), Object::Builtin(b) if std::ptr::eq(Rc::as_ptr(b), site.f));
         if hit && !s.sw.is_null() {
-            let recv = &*base.add(start + 1);
-            let r = match site.kernel {
-                // A container's or string's method with a kernel, on the
-                // receiver in the self slot.
-                Some(kernel) if recv.tag() == site.recv_tag => {
-                    let arg = |k: usize| {
-                        if k <= site.argc {
-                            base.add(start + 1 + k).cast_const()
-                        } else {
-                            std::ptr::null()
-                        }
-                    };
-                    let mut out = std::mem::MaybeUninit::<Object>::uninit();
-                    (kernel(out.as_mut_ptr(), recv, arg(1), arg(2), site.f) == 0)
-                        .then(|| Ok(out.assume_init()))
-                }
-                // (A kernel's kind runs nothing in place on another
-                // receiver: through `h_call`.)
-                Some(_) => None,
-                None if site.in_core => {
-                    let first = if matches!(recv, Object::Unbound) {
-                        start + 2
+            let first = if matches!(&*base.add(start + 1), Object::Unbound) {
+                start + 2
+            } else {
+                start + 1
+            };
+            let args = std::slice::from_raw_parts(base.add(first), lenu - first);
+            let b = &*site.f;
+            // (A body that runs no Python code for any arguments, or for
+            // scalar ones: as `h_call` and `leaf_builtin_call` run them.)
+            let body = || {
+                Some(match b.call_kw.as_ref() {
+                    Some(ckw) => ckw(args, &[]),
+                    None => (b.call)(args),
+                })
+            };
+            let r = match site.kind {
+                Some(LeafKind::Fast(f)) => f(args),
+                Some(LeafKind::Len) => len_fast(args),
+                Some(LeafKind::Isinstance) => Interpreter::core_isinstance(args),
+                Some(LeafKind::Opaque) => body(),
+                Some(LeafKind::Scalar) => {
+                    if args
+                        .iter()
+                        .all(|a| matches!(a, Object::Int(_) | Object::Float(_) | Object::Bool(_)))
+                    {
+                        body()
                     } else {
-                        start + 1
-                    };
-                    let args = std::slice::from_raw_parts(base.add(first), lenu - first);
-                    let b = &*site.f;
-                    // (A body that runs no Python code for any arguments,
-                    // or for scalar ones: as `h_call` and
-                    // `leaf_builtin_call` run them.)
-                    let body = || {
-                        Some(match b.call_kw.as_ref() {
-                            Some(ckw) => ckw(args, &[]),
-                            None => (b.call)(args),
-                        })
-                    };
-                    match site.kind {
-                        Some(LeafKind::Fast(f)) => f(args),
-                        Some(LeafKind::Len) => len_fast(args),
-                        Some(LeafKind::Isinstance) => Interpreter::core_isinstance(args),
-                        Some(LeafKind::Opaque) => body(),
-                        Some(LeafKind::Scalar) => {
-                            if args.iter().all(|a| {
-                                matches!(a, Object::Int(_) | Object::Float(_) | Object::Bool(_))
-                            }) {
-                                body()
-                            } else {
-                                None
-                            }
-                        }
-                        Some(k) => (*s.interp).leaf_builtin_call(k, b, args),
-                        None => None,
+                        None
                     }
                 }
+                Some(k) => (*s.interp).leaf_builtin_call(k, b, args),
                 None => None,
             };
-            if let Some(r) = r {
-                // The operands leave by their releases.
-                crate::release_operands(&*s.interp, base.add(start), lenu - start);
-                return match r {
-                    Ok(v) => {
-                        base.add(start).write(v);
-                        s.len = start + 1;
-                        0
-                    }
-                    Err(e) => {
-                        s.len = start;
-                        s.pc = pc as usize + 1;
-                        s.err = Some(e);
-                        RAISED
-                    }
-                };
+            match r {
+                None => {}
+                Some(Ok(v)) => {
+                    // The operands leave by their releases.
+                    crate::release_operands(&*s.interp, base.add(start), lenu - start);
+                    base.add(start).write(v);
+                    s.len = start + 1;
+                    return 0;
+                }
+                Some(Err(e)) => {
+                    crate::release_operands(&*s.interp, base.add(start), lenu - start);
+                    s.len = start;
+                    s.pc = pc as usize + 1;
+                    s.err = Some(e);
+                    return RAISED;
+                }
             }
         }
         h_call(st, code, ext, pc, len)
@@ -10726,9 +10744,9 @@ impl<'a> Lower<'a> {
 
     /// The helper and its site for a `CALL` (of `argc` arguments) at `pc`
     /// whose site settled on a builtin (see [`BuiltinSite`]): a leaf kind
-    /// with a kernel or one run in place through [`h_call_leaf`], a body
-    /// through the core loop's lane through [`h_call_native`]; `None` for
-    /// anything else (`h_call` alone).
+    /// with a kernel through [`h_call_kernel`], one run in place through
+    /// [`h_call_leaf`], a body through the core loop's lane through
+    /// [`h_call_native`]; `None` for anything else (`h_call` alone).
     fn builtin_site(&mut self, pc: usize, argc: usize) -> Option<(usize, Value)> {
         use crate::LeafKind;
         let (f, kind, via_call) = crate::call_site_builtin(self.ext, pc)?;
@@ -10751,14 +10769,14 @@ impl<'a> Lower<'a> {
         let in_core = kind.is_some_and(LeafKind::runs_in_core);
         let helper = match kind {
             None => h_call_native as *const () as usize,
-            Some(_) if kernel.is_some() || in_core => h_call_leaf as *const () as usize,
+            Some(_) if kernel.is_some() => h_call_kernel as *const () as usize,
+            Some(_) if in_core => h_call_leaf as *const () as usize,
             Some(_) => return None,
         };
         let site = Box::new(BuiltinSite {
             f: Rc::as_ptr(&f),
             argc,
             kind,
-            in_core,
             kernel,
             recv_tag,
             via_call,
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 0ac98e4d..86e6e291 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -73722,40 +73722,49 @@ fn drop_hot(v: Object) {
     }
 }
 
-/// [`Interpreter::release`] of a call's operand with its commonest case in
-/// line: a scalar, or a shared value's plain decrement while nothing
-/// watches releases (one load of [`plain_release_gate`]'s flags). A last
-/// owner, or any release an observer wants, takes the full release.
+/// Whether a shared value's reference may be released by a plain
+/// decrement: one thread owns every count and no observer wants releases
+/// (one load of [`plain_release_gate`]'s flags; a debug build's counts are
+/// always atomic, as [`crate::rc::try_release_word`]'s are).
 #[inline(always)]
-fn release_hot(interp: &Interpreter, v: Object) {
-    if let Some(word) = v.strong_word() {
-        // SAFETY: the gates are four flags laid out as one `u32`; `v` owns
-        // a reference to the live allocation, released here unless it's
-        // the last one.
-        let plain = unsafe {
+fn release_gates_clear() -> bool {
+    // SAFETY: the gates are four flags laid out as one `u32`.
+    !cfg!(debug_assertions)
+        && unsafe {
             (*plain_release_gate().cast::<std::sync::atomic::AtomicU32>())
                 .load(std::sync::atomic::Ordering::Relaxed)
                 == 0
-                && crate::rc::try_release_word(word)
-        };
-        if !plain {
-            return interp.release(v);
         }
-    }
-    std::mem::forget(v);
 }
 
 /// Release the `n` call operands at `p` (moved out: the caller's stack no
-/// longer counts them) in order, as [`release_hot`] does.
+/// longer counts them) in order, as [`Interpreter::release`] does, with
+/// the commonest cases in line: a scalar, or a shared value's plain
+/// decrement while [`release_gates_clear`] holds. A last owner, or any
+/// release an observer wants, takes the full release.
 ///
 /// # Safety
 ///
 /// `p` holds `n` initialized values the caller owns and gives up.
 #[inline(always)]
 unsafe fn release_operands(interp: &Interpreter, p: *const Object, n: usize) {
+    let plain = release_gates_clear();
     for k in 0..n {
-        // SAFETY: the caller's contract.
-        release_hot(interp, unsafe { p.add(k).read() });
+        // SAFETY: the caller's contract: each value owns a reference to its
+        // live allocation, which (with the gates clear) only this thread
+        // counts; the value is read out only for the full release.
+        unsafe {
+            let at = p.add(k);
+            let Some(word) = (*at).strong_word() else {
+                continue;
+            };
+            let c = (*word).load(std::sync::atomic::Ordering::Relaxed);
+            if plain && c > 1 {
+                (*word).store(c - 1, std::sync::atomic::Ordering::Relaxed);
+            } else {
+                interp.release(at.read());
+            }
+        }
     }
 }
 

From 165af404439f0c28641073a1d2a96c5e04fb6aae Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 18:39:46 -0700
Subject: [PATCH 121/173] perf: load exact builtin receivers' methods in line
 and settle bound calls

A compiled method load whose site last resolved an exact list, dict, set
or str receiver's method now checks the receiver's tag in line and pushes
the method (which the code holds) with a count taken, instead of calling
h_load_method: an exact builtin type's methods never change, so the
variant alone keys the method.

The kernel and lane helpers also take a bound method of their site's
builtin with an empty self slot (append = y.append; append(x) in
copy._deepcopy_list, reductor(4) in copy.deepcopy), and the lane settles
x.__reduce_ex__ of a plain instance, which it already ran through
Interpreter::call, so its site stops re-deciding on every call. The lane's
body writes its result into the callee's slot and hands back only an
error, instead of returning the whole result by value.

Instructions per iteration (probe, previous commit -> this one):
d.setdefault + d.pop 1946 -> 1743.

Suite geomean against CPython 1.405 -> 1.403 (0.998x; deepcopy 0.953x,
the rest within quick-build noise).
---
 crates/weavepy-vm/src/frame_jit.rs | 108 +++++++++++++++++++++++++----
 crates/weavepy-vm/src/lib.rs       |  53 ++++++++++----
 2 files changed, 135 insertions(+), 26 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 4dc7a583..eeb837c1 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -5445,12 +5445,23 @@ unsafe extern "C" fn h_call_kernel(
         let lenu = len as usize;
         let start = lenu - site.argc - 2;
         let base = s.stack;
-        let recv = base.add(start + 1);
-        if let (Object::Builtin(b), Some(kernel), false) = (&*base.add(start), site.kernel, s.sw.is_null()) {
-            if std::ptr::eq(Rc::as_ptr(b), site.f) && (*recv).tag() == site.recv_tag {
+        // The receiver: the self slot's, or a bound method's (`append =
+        // y.append`) with the slot empty.
+        let recv: *const Object = match (&*base.add(start), &*base.add(start + 1)) {
+            (Object::Builtin(b), _) if std::ptr::eq(Rc::as_ptr(b), site.f) => base.add(start + 1),
+            (Object::BoundMethod(bm), Object::Unbound)
+                if !bm.redispatch_descriptor
+                    && matches!(&bm.function, Object::Builtin(b) if std::ptr::eq(Rc::as_ptr(b), site.f)) =>
+            {
+                &raw const bm.receiver
+            }
+            _ => std::ptr::null(),
+        };
+        if let (false, Some(kernel), false) = (recv.is_null(), site.kernel, s.sw.is_null()) {
+            if (*recv).tag() == site.recv_tag {
                 let arg = |k: usize| {
                     if k <= site.argc {
-                        recv.add(k).cast_const()
+                        base.add(start + 1 + k).cast_const()
                     } else {
                         std::ptr::null()
                     }
@@ -5550,7 +5561,8 @@ unsafe extern "C" fn h_call_leaf(
 /// [`h_call`] at a site settled on a builtin whose body the core loop's
 /// lane runs (see `Interpreter::core_native_body`), named by `site`: the
 /// callee checked to be that builtin by its address (a method with its
-/// receiver in the self slot, or a function with the slot empty), the
+/// receiver in the self slot, a function with the slot empty, or a bound
+/// method of it with the slot empty), the
 /// frame synced, and the body run on the operands in place. `0` the result
 /// in the callee's slot (the same activation running, nothing pending),
 /// `RAISED` the body raised (likewise), `RELOAD` the core loop goes on
@@ -5579,6 +5591,16 @@ unsafe extern "C" fn h_call_native(
             (Object::Builtin(b), recv) => {
                 std::ptr::eq(Rc::as_ptr(b), site.f) && crate::lane_recv_ok(b, recv, site.via_call)
             }
+            // A bound method held in a local (`reductor = x.__reduce_ex__`).
+            // (A class's bound method may have a natively served half:
+            // `h_call`'s.)
+            (Object::BoundMethod(bm), Object::Unbound)
+                if !bm.redispatch_descriptor && !matches!(&bm.receiver, Object::Type(_)) =>
+            {
+                matches!(&bm.function, Object::Builtin(b)
+                    if std::ptr::eq(Rc::as_ptr(b), site.f)
+                        && crate::lane_recv_ok(b, &bm.receiver, site.via_call))
+            }
             _ => false,
         };
         if !hit || s.sw.is_null() {
@@ -5595,26 +5617,25 @@ unsafe extern "C" fn h_call_native(
             && std::ptr::eq(sw.cur, s.frame)
             && crate::hot_gates::loop_gen() == s.snap_gen;
         match r {
-            Ok(v) => {
-                base.add(start).write(v);
-                frame.stack.set_len(start + 1);
-                if in_place {
-                    s.len = start + 1;
-                    return 0;
-                }
+            // The result in the callee's slot.
+            None if in_place => {
+                s.len = start + 1;
+                0
+            }
+            None => {
                 *sw.last = pcu;
                 if sw.pending.is_none() && (*sw.maybe_dead).get() {
                     sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Marked));
                 }
                 RELOAD
             }
-            Err(e) if in_place => {
+            Some(e) if in_place => {
                 s.len = start;
                 s.pc = pcu + 1;
                 s.err = Some(e);
                 RAISED
             }
-            Err(e) => {
+            Some(e) => {
                 sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Raised(e)));
                 RELOAD
             }
@@ -10468,6 +10489,39 @@ impl<'a> Lower<'a> {
         let cache: Box<MethodCache> = Box::default();
         let cache_at = std::ptr::from_ref::<MethodCache>(&cache) as i64;
         self.method_caches.push(cache);
+        // A site that loaded an exact `list`, `dict`, `set` or `str`
+        // receiver's method: for a receiver of that variant, the method
+        // (the code holds it) with a count taken, the receiver above it.
+        if let Some((want, f)) = self.builtin_method_load(pc) {
+            let recv = self.slot_addr(self.depth - 1);
+            let above = self.slot_addr(self.depth);
+            let tag = self.tag_at(recv);
+            let other = self.b.ins().icmp_imm(IntCC::NotEqual, tag, i64::from(want));
+            self.branch_out(other, helper);
+            let rflag = self.b.ins().iconst(
+                self.ptr,
+                std::ptr::from_ref(crate::sync::rc_shared_flag()) as i64,
+            );
+            let rshared = self.b.ins().uload8(types::I64, FLAGS, rflag, 0);
+            self.branch_out(rshared, helper);
+            let obj = Object::Builtin(f.clone());
+            let btag = obj.tag();
+            // SAFETY: an `Object` is 16 bytes; its second word is the
+            // payload, where a counted builtin's count is.
+            let word = unsafe { std::ptr::from_ref(&obj).cast::<[u64; 2]>().read() }[1];
+            drop(obj);
+            self.held.push(f);
+            let w = self.b.ins().iconst(types::I64, word as i64);
+            let count = self.b.ins().load(types::I64, FLAGS, w, 0);
+            let count = self.b.ins().iadd_imm(count, 1);
+            self.b.ins().store(FLAGS, count, w, 0);
+            self.copy16(above, recv);
+            self.write_tag(recv, btag);
+            self.b.ins().store(FLAGS, w, recv, 8);
+            self.b.ins().jump(done, &[]);
+            self.b.switch_to_block(helper);
+            return self.load_method_helper(pc, cache_at, done);
+        }
         match self.layout {
             // An instance of the class the site last saw, which holds no
             // dictionary and fewer values than the name's position among
@@ -10543,6 +10597,32 @@ impl<'a> Lower<'a> {
             }
         }
         self.b.switch_to_block(helper);
+        self.load_method_helper(pc, cache_at, done)
+    }
+
+    /// The method load at `pc` whose site last loaded a builtin type's
+    /// method for an exact `list`, `dict`, `set` or `str` receiver whose
+    /// count compiled code may take in line: the receiver variant's tag
+    /// and the method.
+    fn builtin_method_load(&self, pc: usize) -> Option<(u8, Rc<crate::object::BuiltinFn>)> {
+        if self.cold {
+            return None;
+        }
+        let (tag, f) = crate::load_method_builtin(self.ext, pc)?;
+        let want = match tag {
+            1 => self.tags.list,
+            2 => self.tags.dict,
+            3 => self.tags.set,
+            4 => self.tags.str,
+            _ => return None,
+        };
+        let counted = Object::Builtin(f.clone()).tag();
+        (counted < 64 && self.tags.rc_counted & (1i64 << counted) != 0).then_some((want, f))
+    }
+
+    /// [`Self::load_method`]'s helper call, in the current block, then the
+    /// join at `done`.
+    fn load_method_helper(&mut self, pc: usize, cache_at: i64, done: Block) -> bool {
         let code = self.code_ptr();
         let ext = self.ext_ptr();
         let pcv = self.b.ins().iconst(types::I64, pc as i64);
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 86e6e291..bc0fa540 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -15143,8 +15143,13 @@ impl Interpreter {
                 }
                 // `super().__init__(...)` in an exception class's own
                 // `__init__`: `BaseException.__init__` on the instance,
-                // through the full handler's dispatch.
-                if exception_init_call(b, recv) {
+                // through the full handler's dispatch; likewise
+                // `x.__reduce_ex__(4)` of a plain instance (`copy`,
+                // `pickle`), which may run `copyreg` code.
+                if exception_init_call(b, recv)
+                    || (matches!(b.name, ".object_reduce_ex" | ".object_reduce")
+                        && matches!(recv, Object::Instance(_)))
+                {
                     return Some(true);
                 }
                 // The names below are dispatched by the interpreter, which
@@ -15189,12 +15194,9 @@ impl Interpreter {
             };
         // SAFETY: the activation is synced, and the shape was checked.
         match unsafe { self.core_native_body(sw, frame, pc, callee_at, b, via_call) } {
-            Ok(v) => {
-                frame.stack.push(v);
-                // SAFETY: the running activation's last-pc slot.
-                unsafe { *sw.last = pc };
-            }
-            Err(e) => sw.pending = Some(CoreExit::Stop(LeafStop::Raised(e))),
+            // SAFETY: the running activation's last-pc slot.
+            None => unsafe { *sw.last = pc },
+            Some(e) => sw.pending = Some(CoreExit::Stop(LeafStop::Raised(e))),
         }
         // A finalizer queued by a dying operand runs before the next
         // instruction, as after the full handler's call.
@@ -15216,8 +15218,9 @@ impl Interpreter {
     /// stack holding them), with the activation waiting on the pending
     /// list: the body may run Python code (an element's `__eq__`, a key
     /// function). Then the operands leave as the full handler's do, the
-    /// arguments first, then the receiver and the callable, and the stack
-    /// ends at `start` (the result is the caller's to place).
+    /// arguments first, then the receiver and the callable, and the result
+    /// takes the callee's slot: `None`, the stack ending past it, or the
+    /// error raised, the stack ending at `start`.
     ///
     /// # Safety
     ///
@@ -15233,7 +15236,7 @@ impl Interpreter {
         start: usize,
         b: *const crate::object::BuiltinFn,
         via_call: bool,
-    ) -> Result<Object, RuntimeError> {
+    ) -> Option<RuntimeError> {
         let len = frame.stack.len();
         // SAFETY: the caller's contract: `start + 2 <= len`; the stack
         // isn't resized while the body runs (nested code runs activations
@@ -15275,7 +15278,14 @@ impl Interpreter {
             release_operands(self, base.add(start + 2), len - start - 2);
             release_operands(self, base.add(start + 1), 1);
             release_operands(self, base.add(start), 1);
-            result
+            match result {
+                Ok(v) => {
+                    base.add(start).write(v);
+                    frame.stack.set_len(start + 1);
+                    None
+                }
+                Err(e) => Some(e),
+            }
         }
     }
 
@@ -73397,6 +73407,25 @@ pub(crate) fn call_site_builtin(
     }
 }
 
+/// The builtin type's method the method load at `pc` last resolved, and
+/// the receiver variant it was resolved for (see [`builtin_recv_tag`]):
+/// an exact builtin receiver's method never changes, so the variant alone
+/// keys it.
+#[cfg(feature = "jit")]
+pub(crate) fn load_method_builtin(
+    ext: &CodeConstObjects,
+    pc: usize,
+) -> Option<(u64, Rc<crate::object::BuiltinFn>)> {
+    let slot = ext.method_slot(pc)?;
+    // SAFETY: GIL-serialized; the borrow ends here.
+    match unsafe { &*slot.0.get() } {
+        (key, MethodSlotFn::Builtin(f)) if key & MethodSlot::BUILTIN_TAG != 0 => {
+            Some((key & !MethodSlot::BUILTIN_TAG, f.clone()))
+        }
+        _ => None,
+    }
+}
+
 /// Whether the `CALL` at `pc` last settled on a leaf builtin the core
 /// loop's arm (and the frame JIT's helper) runs in place.
 #[cfg(feature = "jit")]

From 4988dba67472371d014321c1f22cc0424f04e314 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 19:48:01 -0700
Subject: [PATCH 122/173] perf: give more leaf container methods kernels

list.insert (an int index), list.reverse and list.copy, dict.keys,
dict.values and dict.items, set.pop and str.format (arguments that render
without Python dispatch) now have kernels, the same admissions
leaf_builtin_call makes, so compiled code runs them on the operands in
place: a local receiver's call through the fused in-line kernel instead
of h_local_method, and any other through h_call_kernel.

list.insert's body now converts its index as CPython's
PyNumber_AsSsize_t does: a bool or an object with __index__ is accepted,
and an int past Py_ssize_t raises OverflowError (it raised TypeError for
anything but an exact int, including the bools the leaf path admits).

Suite geomean against CPython 1.403 -> 1.398 (0.997x; no workload moved by
more than 1.2%).
---
 crates/weavepy-vm/src/builtins.rs             |  11 +-
 crates/weavepy-vm/src/frame_jit.rs            |  59 ++++++++-
 .../regrtest/test_frame_jit_builtin_sites.py  | 123 ++++++++++++++++++
 3 files changed, 186 insertions(+), 7 deletions(-)

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index ee88aead..00a6a150 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -13179,11 +13179,18 @@ fn list_insert(args: &[Object]) -> Result<Object, RuntimeError> {
     if args.len() != 3 {
         return Err(type_error("list.insert() expected 2 arguments"));
     }
+    let l = list_self(args)?;
+    // CPython converts the index with `PyNumber_AsSsize_t`: `__index__`,
+    // an `OverflowError` past `Py_ssize_t`.
     let i = match &args[1] {
         Object::Int(i) => *i,
-        _ => return Err(type_error("insert index must be int")),
+        Object::Long(_) => {
+            return Err(crate::error::overflow_error(
+                "Python int too large to convert to C ssize_t",
+            ))
+        }
+        other => coerce_index_i64(other)?,
     };
-    let l = list_self(args)?;
     let mut l = l.borrow_mut();
     let len = l.len() as i64;
     let idx = if i < 0 {
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index eeb837c1..fd079b6c 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1778,7 +1778,11 @@ fn kernel_for(tag: u64, kind: crate::LeafKind, nargs: usize) -> Option<Kernel> {
         (1, K::ListAppend, 1) => k_list_append,
         (1, K::ListPop, 0) => k_list_pop,
         (1, K::ListExtend, 1) => k_list_extend,
+        (1, K::ListInsert, 2) => k_list_insert,
+        (1, K::ListReverse | K::ListCopy, 0) => k_body,
         (2, K::DictGet, 1 | 2) => k_dict_get,
+        (2, K::DictKeys | K::DictValues | K::DictItems, 0) => k_body,
+        (3, K::SetPop, 0) => k_body,
         (3, K::SetAdd, 1) => k_set_add,
         (3, K::SetDiscard, 1) => k_set_discard,
         (3, K::SetRemove, 1) => k_set_remove,
@@ -1788,12 +1792,13 @@ fn kernel_for(tag: u64, kind: crate::LeafKind, nargs: usize) -> Option<Kernel> {
         (4, K::StrIsalpha, 0) => k_str_isalpha,
         (4, K::StrIsspace, 0) => k_str_isspace,
         (4, K::StrFind, 1 | 2) => k_str_find,
-        (4, K::StrLower | K::StrUpper, 0) => k_str_body,
+        (4, K::StrLower | K::StrUpper, 0) => k_body,
         (4, K::StrStrip | K::StrLstrip | K::StrRstrip, 0 | 1) => k_str_strip,
         (4, K::StrSplit, 0 | 1) => k_str_split,
         (4, K::StrJoin, 1) => k_str_join,
         (4, K::StrReplace, 2) => k_str_replace,
-        (4, K::Opaque, _) => k_str_body,
+        (4, K::StrFormat, 0..=2) => k_str_format,
+        (4, K::Opaque, _) => k_body,
         _ => return None,
     })
 }
@@ -2222,9 +2227,11 @@ unsafe extern "C" fn k_str_find(
     }
 }
 
-/// A `str` method whose body runs no Python code for any arguments
-/// (`lower`, `upper`, the registered pure bodies).
-unsafe extern "C" fn k_str_body(
+/// A method whose body runs no Python code for any arguments on its
+/// receiver variant (`str.lower`, the registered pure `str` bodies,
+/// `list.reverse`, `dict.items`, `set.pop`, which hands its element's
+/// reference over).
+unsafe extern "C" fn k_body(
     out: *mut Object,
     recv: *const Object,
     a: *const Object,
@@ -2235,6 +2242,48 @@ unsafe extern "C" fn k_str_body(
     unsafe { run_body(out, recv, a, b, f) }
 }
 
+/// `lst.insert(i, x)` with an `int` index, whose conversion runs no code.
+unsafe extern "C" fn k_list_insert(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    f: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe {
+        if !matches!(&*a, Object::Int(_) | Object::Bool(_)) {
+            return 1;
+        }
+        run_body(out, recv, a, b, f)
+    }
+}
+
+/// `"...{}...".format(x, ...)` of arguments that render without Python
+/// dispatch (as `leaf_builtin_call` admits them).
+unsafe extern "C" fn k_str_format(
+    out: *mut Object,
+    recv: *const Object,
+    a: *const Object,
+    b: *const Object,
+    f: *const crate::object::BuiltinFn,
+) -> u32 {
+    // SAFETY: as `Kernel`'s.
+    unsafe {
+        let plain = |v: *const Object| {
+            v.is_null()
+                || matches!(
+                    &*v,
+                    Object::Str(_) | Object::Int(_) | Object::Bool(_) | Object::Float(_) | Object::None
+                )
+        };
+        if !(plain(a) && plain(b)) {
+            return 1;
+        }
+        run_body(out, recv, a, b, f)
+    }
+}
+
 unsafe extern "C" fn k_str_strip(
     out: *mut Object,
     recv: *const Object,
diff --git a/tests/regrtest/test_frame_jit_builtin_sites.py b/tests/regrtest/test_frame_jit_builtin_sites.py
index c7ef275b..31531325 100644
--- a/tests/regrtest/test_frame_jit_builtin_sites.py
+++ b/tests/regrtest/test_frame_jit_builtin_sites.py
@@ -182,6 +182,88 @@ def profiled(n, events):
     return t
 
 
+def copies(xs):
+    out = []
+    for x in xs:
+        out.append(x.copy())
+    return out
+
+
+class ItemsDict(dict):
+    def items(self):
+        return [("overridden", 1)]
+
+
+def all_items(ds):
+    out = []
+    for d in ds:
+        for k, v in d.items():
+            out.append((k, v))
+        out.append(len(d.keys()) + len(d.values()))
+    return out
+
+
+class Index:
+    def __init__(self, i):
+        self.i = i
+
+    def __index__(self):
+        return self.i
+
+
+def inserts(n):
+    lst = []
+    bad = 0
+    for i in range(n):
+        lst.insert(0, i)
+        lst.insert(True, -i)
+        lst.insert(Index(-1), i * 2)
+        lst.reverse()
+        try:
+            lst.insert("0", i)
+        except TypeError:
+            bad += 1
+    return lst, bad
+
+
+class Fmt:
+    def __format__(self, spec):
+        return "fmt"
+
+
+def formats(n):
+    out = []
+    for i in range(n):
+        tmpl = "{}-{}"
+        out.append(tmpl.format(i, "x"))
+        out.append(tmpl.format(Fmt(), 1.5))
+    return out
+
+
+def set_pops(n):
+    taken = 0
+    s = set()
+    for i in range(n):
+        s.add(i)
+        taken += s.pop() == i
+        try:
+            s.pop()
+        except KeyError as e:
+            taken += e.__traceback__.tb_lineno == SET_POP_LINE
+    return taken
+
+
+SET_POP_LINE = set_pops.__code__.co_firstlineno + 7
+
+
+def bound_appends(n):
+    y = []
+    append = y.append
+    for i in range(n):
+        append(i)
+    return y
+
+
 class BuiltinSiteTest(unittest.TestCase):
     def test_leaf_and_rebinding(self):
         self.assertEqual(lengths([1, 2], N), 2 * N)
@@ -260,6 +342,47 @@ def test_profile_from_compiled_code(self):
         self.assertEqual(calls.count("len"), N - N // 2)
         self.assertEqual(calls.count("count"), N - N // 2)
 
+    def test_method_load_variants(self):
+        lists = [[i] for i in range(N)]
+        self.assertEqual(copies(lists), lists)
+        mixed = [[1], {"a": 1}, {2}, "s"] * 50
+        try:
+            copies(mixed)
+        except AttributeError as e:
+            self.assertIn("'str' object has no attribute 'copy'", str(e))
+        else:
+            self.fail("no AttributeError")
+        out = copies([[1], {"a": 1}, {2}] * 50)
+        self.assertEqual(out[:3], [[1], {"a": 1}, {2}])
+        self.assertEqual([type(x) for x in out[:3]], [list, dict, set])
+
+    def test_dict_views(self):
+        ds = [{"a": 1, "b": 2}] * N + [ItemsDict(z=3)]
+        out = all_items(ds)
+        self.assertEqual(out[:3], [("a", 1), ("b", 2), 4])
+        self.assertEqual(out[-2:], [("overridden", 1), 2])
+
+    def test_list_insert_reverse(self):
+        got, bad = inserts(300)
+        want = []
+        for i in range(300):
+            want.insert(0, i)
+            want.insert(1, -i)
+            want.insert(-1, i * 2)
+            want.reverse()
+        self.assertEqual(got, want)
+        self.assertEqual(bad, 300)
+
+    def test_str_format(self):
+        out = formats(N)
+        self.assertEqual(out[:4], ["0-x", "fmt-1.5", "1-x", "fmt-1.5"])
+
+    def test_set_pop(self):
+        self.assertEqual(set_pops(N), 2 * N)
+
+    def test_bound_method_kernel(self):
+        self.assertEqual(bound_appends(N), list(range(N)))
+
     def test_forced_frame_jit(self):
         if FORCED in sys.argv:
             self.skipTest("already forced")

From 545438daa42b423a2922d1a5f155b064e5490bb2 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 20:41:35 -0700
Subject: [PATCH 123/173] perf: call a compiled site's settled builtin type
 through its own helper

A CALL site that constructed a builtin type in place (type(x), int(s),
str(n), an empty list() or dict(), a builtin exception) now remembers the
type in its method slot, from the core loop's arm or h_call. Compiled code
for such a site calls h_call_type, which checks the callee against the
type by identity and runs the in-place constructor
(Interpreter::leaf_type_call, then seqiter::builtin_ctor_pure) on the
operands directly, releasing them after, instead of h_call's chain of
generator, leaf-builtin, Python-callee and native-class checks. Anything
else goes on to h_call.

Instructions per iteration with tier 2 off (type(x) is int, len(list()),
int("7")): 3451 -> 3190.

Suite geomean against CPython 1.398 -> 1.398 on the host (chaos 0.968x,
deepcopy 0.981x; decimal_telco 1.021x and raytrace 1.015x there, while
callgrind in the Linux container measures decimal_telco at 0.988x of the
previous commit).
---
 crates/weavepy-vm/src/frame_jit.rs            | 68 ++++++++++++++++++-
 crates/weavepy-vm/src/lib.rs                  | 44 ++++++++++++
 .../regrtest/test_frame_jit_builtin_sites.py  | 52 ++++++++++++++
 3 files changed, 163 insertions(+), 1 deletion(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index fd079b6c..acffbfff 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -212,6 +212,8 @@ pub(crate) struct Native {
     /// The `CALL` sites' settled builtins.
     #[allow(clippy::vec_box)]
     _builtin_sites: Vec<Box<BuiltinSite>>,
+    #[allow(clippy::vec_box)]
+    _type_sites: Vec<Box<TypeSite>>,
     /// The builtins the method kernels' calls address.
     _held: Vec<Rc<crate::object::BuiltinFn>>,
     #[allow(clippy::vec_box)]
@@ -5228,6 +5230,7 @@ unsafe extern "C" fn h_call(
                 _ => None,
             };
             if let Some(r) = r {
+                crate::note_type_site(Some(ext), pc, &ops[0]);
                 return done(st, r);
             }
         }
@@ -5240,6 +5243,7 @@ unsafe extern "C" fn h_call(
         if let Object::Type(t) = &ops[0] {
             if t.flags.is_builtin {
                 if let Some(r) = interp.core_builtin_ctor(base, start, len) {
+                    crate::note_type_site(Some(ext), pc, &*base.add(start));
                     return done(st, r);
                 }
                 if argc == 1 && !st.sw.is_null() {
@@ -5607,6 +5611,52 @@ unsafe extern "C" fn h_call_leaf(
     }
 }
 
+/// A `CALL` site that constructed a builtin type in place when its code
+/// compiled (see `Lower::call_op`): the type and the operand count.
+pub(crate) struct TypeSite {
+    cls: Rc<crate::types::TypeObject>,
+    argc: usize,
+}
+
+/// [`h_call`] at a site settled on a builtin type `site` names: the callee
+/// checked to be that type, with an empty self slot, and its constructor
+/// run in place when nothing it does can run Python code (as
+/// `Interpreter::core_builtin_ctor` runs it), the operands released after;
+/// anything else through `h_call`.
+unsafe extern "C" fn h_call_type(
+    st: *mut State,
+    code: *const CodeObject,
+    ext: *const CodeConstObjects,
+    pc: u64,
+    len: u64,
+    site: *const TypeSite,
+) -> u32 {
+    // SAFETY: as `h_call`'s; the site is this `CALL`'s.
+    unsafe {
+        let s = &mut *st;
+        let site = &*site;
+        let lenu = len as usize;
+        let start = lenu - site.argc - 2;
+        let base = s.stack;
+        let hit = matches!(&*base.add(start), Object::Type(t) if Rc::ptr_eq(t, &site.cls))
+            && matches!(&*base.add(start + 1), Object::Unbound);
+        if hit && !s.sw.is_null() {
+            let args = std::slice::from_raw_parts(base.add(start + 2), site.argc);
+            let interp = &*s.interp;
+            if let Some(v) = interp
+                .leaf_type_call(&site.cls, args)
+                .or_else(|| crate::seqiter::builtin_ctor_pure(&site.cls, args))
+            {
+                crate::release_operands(interp, base.add(start), lenu - start);
+                base.add(start).write(v);
+                s.len = start + 1;
+                return 0;
+            }
+        }
+        h_call(st, code, ext, pc, len)
+    }
+}
+
 /// [`h_call`] at a site settled on a builtin whose body the core loop's
 /// lane runs (see `Interpreter::core_native_body`), named by `site`: the
 /// callee checked to be that builtin by its address (a method with its
@@ -6720,6 +6770,7 @@ fn compile_with(
     let mut methods: Vec<Box<MethodCache>>;
     let mut direct: Vec<Box<DirectSite>>;
     let mut builtin_sites: Vec<Box<BuiltinSite>>;
+    let mut type_sites: Vec<Box<TypeSite>>;
     let mut held: Vec<Rc<crate::object::BuiltinFn>>;
     let mut mods: Vec<Box<ModCache>>;
     let t0 = stats::enabled().then(std::time::Instant::now);
@@ -6738,6 +6789,7 @@ fn compile_with(
         methods = std::mem::take(&mut lower.method_caches);
         direct = std::mem::take(&mut lower.direct_sites);
         builtin_sites = std::mem::take(&mut lower.builtin_sites);
+        type_sites = std::mem::take(&mut lower.type_sites);
         held = std::mem::take(&mut lower.held);
         mods = std::mem::take(&mut lower.mod_caches);
         // (Measured as if every exit's tail were in line, the size the
@@ -6809,6 +6861,7 @@ fn compile_with(
         _methods: methods,
         _direct: direct,
         _builtin_sites: builtin_sites,
+        _type_sites: type_sites,
         _held: held,
         _mods: mods,
     })
@@ -6924,6 +6977,9 @@ struct Lower<'a> {
     /// The `CALL` sites' settled builtins (see [`BuiltinSite`]).
     #[allow(clippy::vec_box)]
     builtin_sites: Vec<Box<BuiltinSite>>,
+    /// The `CALL` sites' settled builtin types (see [`TypeSite`]).
+    #[allow(clippy::vec_box)]
+    type_sites: Vec<Box<TypeSite>>,
     /// The builtins the method kernels' calls address.
     held: Vec<Rc<crate::object::BuiltinFn>>,
     /// The method loads' module-attribute caches.
@@ -7107,6 +7163,7 @@ impl<'a> Lower<'a> {
             method_caches: Vec::new(),
             direct_sites: Vec::new(),
             builtin_sites: Vec::new(),
+            type_sites: Vec::new(),
             held: Vec::new(),
             mod_caches: Vec::new(),
             catch: None,
@@ -10878,7 +10935,16 @@ impl<'a> Lower<'a> {
     /// [`h_call_native`]; `None` for anything else (`h_call` alone).
     fn builtin_site(&mut self, pc: usize, argc: usize) -> Option<(usize, Value)> {
         use crate::LeafKind;
-        let (f, kind, via_call) = crate::call_site_builtin(self.ext, pc)?;
+        let Some((f, kind, via_call)) = crate::call_site_builtin(self.ext, pc) else {
+            let cls = crate::call_site_type(self.ext, pc)?;
+            let site = Box::new(TypeSite { cls, argc });
+            let at = self
+                .b
+                .ins()
+                .iconst(self.ptr, std::ptr::from_ref::<TypeSite>(&site) as i64);
+            self.type_sites.push(site);
+            return Some((h_call_type as *const () as usize, at));
+        };
         let (kernel, recv_tag) = match kind {
             // (An opaque body's kernel is the body: run in place on any
             // receiver below.)
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index bc0fa540..1b94db85 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -13551,6 +13551,8 @@ impl Interpreter {
                             .flatten()
                         {
                             let callee_at = len - argc - 2;
+                            // SAFETY: as above: the callee.
+                            note_type_site(ext, pc, unsafe { &*base.add(callee_at) });
                             // SAFETY: every operand was checked droppable;
                             // the result takes the callee's slot.
                             unsafe {
@@ -73426,6 +73428,28 @@ pub(crate) fn load_method_builtin(
     }
 }
 
+/// The builtin type the `CALL` at `pc` last constructed in place.
+#[cfg(feature = "jit")]
+pub(crate) fn call_site_type(ext: &CodeConstObjects, pc: usize) -> Option<Rc<crate::types::TypeObject>> {
+    let slot = ext.method_slot(pc)?;
+    // SAFETY: GIL-serialized; the borrow ends here.
+    match unsafe { &(*slot.0.get()).1 } {
+        MethodSlotFn::BuiltinType(t) => Some(t.clone()),
+        _ => None,
+    }
+}
+
+/// Note at the `CALL` site at `pc` that its callee, a builtin type, was
+/// constructed in place (see [`MethodSlot::note_builtin_type`]).
+#[inline]
+fn note_type_site(ext: Option<&CodeConstObjects>, pc: usize, callee: &Object) {
+    if let (Some(slot), Object::Type(t)) = (ext.and_then(|e| e.method_slot(pc)), callee) {
+        if t.flags.is_builtin && t.native_kind.get() == 0 {
+            slot.note_builtin_type(t);
+        }
+    }
+}
+
 /// Whether the `CALL` at `pc` last settled on a leaf builtin the core
 /// loop's arm (and the frame JIT's helper) runs in place.
 #[cfg(feature = "jit")]
@@ -73960,6 +73984,10 @@ enum MethodSlotFn {
         generation: u64,
         fast: leaf_builtins::Fast,
     },
+    /// A `CALL` site's builtin type (`type(x)`, `int(s)`, `list()`), which
+    /// the site constructed in place (see `Interpreter::core_builtin_ctor`).
+    /// Builtin types live as long as the interpreter.
+    BuiltinType(Rc<crate::types::TypeObject>),
     /// A rejected native bound-call body, weakly held so a call site
     /// doesn't extend its lifetime. The key is the registry generation.
     NonLeaf(crate::sync::Weak<crate::object::BoundMethod>),
@@ -74295,6 +74323,22 @@ impl MethodSlot {
         unsafe { *self.0.get() = (0, MethodSlotFn::NativeBody(f.clone(), via_call)) };
     }
 
+    /// Remember that this `CALL` site constructed the builtin type `t` in
+    /// place, unless the site already settled on another callee.
+    #[inline]
+    fn note_builtin_type(&self, t: &Rc<crate::types::TypeObject>) {
+        // SAFETY: as `get`; the shared borrow ends before the write.
+        let vacant = match unsafe { &*self.0.get() } {
+            (_, MethodSlotFn::BuiltinType(c)) => !Rc::ptr_eq(c, t),
+            (0, MethodSlotFn::Py(w)) => w.strong_count() == 0,
+            _ => false,
+        };
+        if vacant {
+            // SAFETY: as `set`.
+            unsafe { *self.0.get() = (0, MethodSlotFn::BuiltinType(t.clone())) };
+        }
+    }
+
     fn is_non_leaf(&self, method: &Rc<crate::object::BoundMethod>) -> bool {
         // SAFETY: GIL-held leaf dispatch, as `get_leaf`. A live Weak keeps
         // the allocation reserved, preventing address-reuse aliasing.
diff --git a/tests/regrtest/test_frame_jit_builtin_sites.py b/tests/regrtest/test_frame_jit_builtin_sites.py
index 31531325..271e8780 100644
--- a/tests/regrtest/test_frame_jit_builtin_sites.py
+++ b/tests/regrtest/test_frame_jit_builtin_sites.py
@@ -264,6 +264,46 @@ def bound_appends(n):
     return y
 
 
+def type_calls(xs):
+    out = []
+    for x in xs:
+        out.append((type(x).__name__, len(list()), str(1), int("5")))
+    return out
+
+
+def rebound_type(n):
+    global type
+    out = []
+    try:
+        for i in range(n):
+            if i == n // 2:
+                type = lambda x: "rebound"  # noqa: E731
+            out.append(type(i))
+    finally:
+        del type
+    return out
+
+
+def bad_ints(strs):
+    out = []
+    for s in strs:
+        try:
+            out.append(int(s))
+        except ValueError as e:
+            out.append((str(e), e.__traceback__.tb_lineno))
+    return out
+
+
+BAD_INT_LINE = bad_ints.__code__.co_firstlineno + 4
+
+
+def errors(n):
+    out = []
+    for i in range(n):
+        out.append(ValueError("v", i))
+    return out
+
+
 class BuiltinSiteTest(unittest.TestCase):
     def test_leaf_and_rebinding(self):
         self.assertEqual(lengths([1, 2], N), 2 * N)
@@ -383,6 +423,18 @@ def test_set_pop(self):
     def test_bound_method_kernel(self):
         self.assertEqual(bound_appends(N), list(range(N)))
 
+    def test_type_sites(self):
+        out = type_calls([1, "s", 2.0] * 1000)
+        self.assertEqual(out[:3], [("int", 0, "1", 5), ("str", 0, "1", 5), ("float", 0, "1", 5)])
+        out = rebound_type(N)
+        self.assertEqual(out[: N // 2], [int] * (N // 2))
+        self.assertEqual(out[N // 2 :], ["rebound"] * (N - N // 2))
+        out = bad_ints(["1", "x", "3"] * 1000)
+        self.assertEqual(out[:3], [1, ("invalid literal for int() with base 10: 'x'", BAD_INT_LINE), 3])
+        errs = errors(N)
+        self.assertEqual([type(e) for e in errs[:2]], [ValueError, ValueError])
+        self.assertEqual(errs[5].args, ("v", 5))
+
     def test_forced_frame_jit(self):
         if FORCED in sys.argv:
             self.skipTest("already forced")

From f7821e186b23f547fde558e4079d004977474a27 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 21:32:42 -0700
Subject: [PATCH 124/173] perf: check and unwrap a builtin subclass receiver
 without allocating

A builtin type's method called on an instance of a subclass (a list or
dict subclass, collections.Counter, defaultdict) runs through the method
table's unwrap shim, which checked the receiver's class through a cloned
class and the generic class_of, then copied the whole argument slice into
a new vector to swap in the native payload. The receiver check now
borrows an instance's class and compares it with the owner by address
before walking the MRO, and the payload goes in front of bitwise views of
the other arguments in a stack buffer.

Instructions per iteration (probe, previous commit -> this one): list
subclass append + pop 2299 -> 2086.

Suite geomean against CPython 1.398 -> 1.400 on the host (1.0015x, within
quick-build noise: callgrind measures sorting at 0.996x and
collections_mix at 1.001x of the previous commit).
---
 crates/weavepy-vm/src/type_surface.rs         | 43 ++++++++++++++++---
 .../regrtest/test_frame_jit_builtin_sites.py  | 40 +++++++++++++++++
 2 files changed, 77 insertions(+), 6 deletions(-)

diff --git a/crates/weavepy-vm/src/type_surface.rs b/crates/weavepy-vm/src/type_surface.rs
index 7835d39f..56231afb 100644
--- a/crates/weavepy-vm/src/type_surface.rs
+++ b/crates/weavepy-vm/src/type_surface.rs
@@ -1179,6 +1179,30 @@ fn as_native(o: &Object) -> Object {
     o.clone()
 }
 
+/// `body` on `args` with `recv` in place of the first: the other
+/// arguments as bitwise views (never dropped: `args` keeps them alive, and
+/// a body clones what it keeps), without allocating for the usual few.
+fn with_receiver<R>(args: &[Object], recv: Object, body: impl FnOnce(&[Object]) -> R) -> R {
+    const N: usize = 8;
+    if args.len() > N {
+        let mut v = args.to_vec();
+        v[0] = recv;
+        return body(&v);
+    }
+    let mut buf = [const { std::mem::MaybeUninit::<Object>::uninit() }; N];
+    buf[0].write(recv);
+    for (slot, a) in buf[1..].iter_mut().zip(&args[1..]) {
+        // SAFETY: a view of a live argument, never dropped.
+        slot.write(unsafe { std::ptr::read(a) });
+    }
+    // SAFETY: the first `args.len()` entries were written.
+    let view = unsafe { std::slice::from_raw_parts(buf.as_ptr().cast::<Object>(), args.len()) };
+    let r = body(view);
+    // SAFETY: the receiver is the one owned entry.
+    unsafe { buf[0].assume_init_drop() };
+    r
+}
+
 /// Wrap a `lookup_method`-table builtin so an `Instance` receiver
 /// (built-in subclass) is unwrapped to its native payload before the
 /// underlying implementation runs.
@@ -1199,6 +1223,17 @@ fn unwrap_shim(inner: Rc<BuiltinFn>, owner: &Rc<TypeObject>) -> Object {
         owner: &crate::sync::Weak<TypeObject>,
         first: &Object,
     ) -> Result<(), RuntimeError> {
+        // An instance (a subclass's) of the owner, its class borrowed: the
+        // commonest receiver here. (A live weak handle keeps its
+        // allocation, so the owner's address names it.)
+        if let Object::Instance(inst) = first {
+            let cls = inst.class.borrow();
+            if std::ptr::eq(Rc::as_ptr(&cls), owner.as_ptr())
+                || owner.upgrade().is_some_and(|o| cls.is_subclass_of(&o))
+            {
+                return Ok(());
+            }
+        }
         if let Some(owner) = owner.upgrade() {
             let cls = crate::builtins::class_of(first);
             if !cls.is_subclass_of(&owner) {
@@ -1235,9 +1270,7 @@ fn unwrap_shim(inner: Rc<BuiltinFn>, owner: &Rc<TypeObject>) -> Object {
                 if !keep_instance {
                     let unwrapped = as_native(first);
                     if !unwrapped.is_same(first) {
-                        let mut v = args.to_vec();
-                        v[0] = unwrapped;
-                        return (inner_pos.call)(&v);
+                        return with_receiver(args, unwrapped, |v| (inner_pos.call)(v));
                     }
                 }
             }
@@ -1258,9 +1291,7 @@ fn unwrap_shim(inner: Rc<BuiltinFn>, owner: &Rc<TypeObject>) -> Object {
                 }
                 let unwrapped = as_native(first);
                 if !unwrapped.is_same(first) {
-                    let mut v = args.to_vec();
-                    v[0] = unwrapped;
-                    return kw(&v, kwargs);
+                    return with_receiver(args, unwrapped, |v| kw(v, kwargs));
                 }
             }
             kw(args, kwargs)
diff --git a/tests/regrtest/test_frame_jit_builtin_sites.py b/tests/regrtest/test_frame_jit_builtin_sites.py
index 271e8780..5967b730 100644
--- a/tests/regrtest/test_frame_jit_builtin_sites.py
+++ b/tests/regrtest/test_frame_jit_builtin_sites.py
@@ -304,6 +304,38 @@ def errors(n):
     return out
 
 
+class MyDict(dict):
+    pass
+
+
+def subclass_methods(n):
+    import collections
+
+    d = MyDict(a=1)
+    c = collections.Counter("abca")
+    dd = collections.defaultdict(list)
+    out = 0
+    for i in range(n):
+        out += d.get("a", 0) + d.get("z", 5)
+        d.setdefault("k%d" % (i % 3), i)
+        out += c.get("a")
+        dd["x"].append(i)
+        out += len(dd.get("x"))
+    return out, sorted(d), len(dd["x"])
+
+
+def unbound_descriptors(n):
+    bad = 0
+    lst = MyList()
+    for i in range(n):
+        list.append(lst, i)
+        try:
+            list.append({}, i)
+        except TypeError as e:
+            bad += "doesn't apply to a 'dict' object" in str(e)
+    return len(lst), bad
+
+
 class BuiltinSiteTest(unittest.TestCase):
     def test_leaf_and_rebinding(self):
         self.assertEqual(lengths([1, 2], N), 2 * N)
@@ -435,6 +467,14 @@ def test_type_sites(self):
         self.assertEqual([type(e) for e in errs[:2]], [ValueError, ValueError])
         self.assertEqual(errs[5].args, ("v", 5))
 
+    def test_subclass_receivers(self):
+        out, keys, n = subclass_methods(300)
+        want = sum(1 + 5 + 2 + (i + 1) for i in range(300))
+        self.assertEqual(out, want)
+        self.assertEqual(keys, ["a", "k0", "k1", "k2"])
+        self.assertEqual(n, 300)
+        self.assertEqual(unbound_descriptors(300), (300, 300))
+
     def test_forced_frame_jit(self):
         if FORCED in sys.argv:
             self.skipTest("already forced")

From 95214379acdde5326a88ac65a291edeb660842fb Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 23:01:36 -0700
Subject: [PATCH 125/173] perf: hand core-loop instructions over without
 leaving the loop

The core loop's out-of-line instructions (helper cache hits, leaf calls
and lanes) and its handoffs to the full leaf arms used to end the core
loop: leaf_core synced and returned, leaf_burst dispatched the exit, and
the next instruction paid a full reload prologue. Both now run from
inside the loop (core_offload_in_loop), which goes on in place when the
activation, its operand stack and the pending state are unchanged, and
reloads otherwise, as before.

Also in the loop's arms:
- STORE_FAST and DELETE_FAST release a shared old value with an in-line
  decrement (release_hot) instead of calling release.
- The release observers' flags are tested with one 32-bit load
  (sync::release_gates).
- Scalar copies move two opaque words (LLVM had split the enum copy
  into field moves through the stack), as does the first operand of the
  fused LOAD_FAST pair.
- BINARY_OP tests the left operand's tag before the in-place str
  append's checks.

Also fixes dictmap's replace_index_unique, which named Object where its
table's value type V belongs (the crate did not build).

Measured (quick build, instructions retired, PYTHONHASHSEED=0):
- suite geomean 0.990x of the base (every workload improved; sudoku
  0.972x, chaos 0.974x, raytrace 0.979x, tomllib_loads 0.982x,
  stdlib_utils 0.987x, logging_mix 0.991x).
- frame-JIT-off probes, per statement: x = 'abc' 160 -> 119,
  x = a + b 212 -> 198, x = [a, b] 857 -> 778, x = d['k'] 397 -> 377,
  x = G 155 -> 145.
---
 crates/weavepy-vm/src/lib.rs  | 440 ++++++++++++++++++++++------------
 crates/weavepy-vm/src/sync.rs |  17 ++
 2 files changed, 304 insertions(+), 153 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 1b94db85..0466da01 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -2567,6 +2567,28 @@ impl Interpreter {
         }
     }
 
+    /// [`Self::release`] with a shared value's plain decrement in line (the
+    /// core loop's arms that end a binding): only a last owner's release,
+    /// or one an observer watches, takes the full path.
+    #[inline(always)]
+    fn release_hot(&self, dropped: Object) {
+        let Some(word) = dropped.strong_word() else {
+            return std::mem::forget(dropped);
+        };
+        // A clear gate word: the bias holds (this thread alone updates
+        // counts) and no observer watches releases.
+        if crate::sync::release_gates() == 0 {
+            // SAFETY: `dropped` owns a reference to the live allocation.
+            let w = unsafe { &*word };
+            let c = w.load(std::sync::atomic::Ordering::Relaxed);
+            if c != 1 {
+                w.store(c - 1, std::sync::atomic::Ordering::Relaxed);
+                return std::mem::forget(dropped);
+            }
+        }
+        self.release(dropped);
+    }
+
     /// Release a finished call's arguments now, before the caller handles
     /// the result (CPython's `CALL` decrefs its operands first): an
     /// argument only the call held dies here, finalizers and all.
@@ -10859,117 +10881,180 @@ impl Interpreter {
             let exit = self.leaf_core::<false>(sw, snap_gen);
             match exit {
                 #[cfg(feature = "jit")]
-                CoreExit::Switch => {
-                    native = !native;
-                    continue;
-                }
+                CoreExit::Switch => native = !native,
                 CoreExit::Stop(stop) => return stop,
-                CoreExit::Helper => {
-                    // SAFETY: see `CoreSwitch` (the running activation's
-                    // handles; nothing else borrows them here).
-                    let frame = unsafe { &mut *sw.cur };
-                    if burst_stats::enabled() {
-                        if let Some(ins) = frame.code.instructions.get(frame.pc as usize) {
-                            burst_stats::note_helper(ins.op);
-                            let recv = frame.stack.last().map_or("-", |v| v.type_name());
-                            burst_stats::note_site(&frame.code.qualname, frame.pc, ins.op, recv);
-                        }
-                    }
-                    // A cached property site skips the helper's resolution: a
-                    // pure leaf getter is evaluated in place, any other runs
-                    // as an inline activation.
-                    let pc = frame.pc as usize;
-                    if frame
-                        .code
-                        .instructions
-                        .get(pc)
-                        .is_some_and(|i| i.op == OpCode::LoadAttr)
-                    {
-                        if let Some((getter, code)) = Self::cached_property_getter(frame, pc) {
-                            // SAFETY: as above.
-                            let last_pc = unsafe { &mut *sw.last };
-                            if self.core_leaf_property(frame, last_pc, pc, &getter, &code) {
-                                // SAFETY: the running thread's own flag.
-                                if unsafe { (*sw.maybe_dead).get() } {
-                                    return LeafStop::Marked;
-                                }
-                                continue;
-                            }
-                            if self.core_property(sw) {
-                                continue;
-                            }
-                        }
+                CoreExit::Slow => {
+                    if let Some(stop) = self.core_offload(sw, snap_gen, false) {
+                        return stop;
                     }
-                    // SAFETY: as above (`core_property` declined untouched).
-                    let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) };
-                    match self.leaf_core_helper(frame, last_pc) {
-                        // A released receiver or argument may have queued
-                        // a finalizer (see `after_release!` in the core).
+                }
+                CoreExit::Reload => unreachable!("handled inside the core loop"),
+            }
+        }
+    }
+
+    /// Run the instruction the core loop handed over at `sw`'s (synced)
+    /// running activation's pc: out of line (`helper`: the fused shapes,
+    /// cache hits and leaf calls of [`Self::leaf_core_helper`] and the
+    /// lanes) or with the full leaf arms. `None` when the core loop goes on
+    /// (on the activation `sw` now runs), else how the burst ends.
+    #[inline(never)]
+    fn core_offload(
+        &mut self,
+        sw: &mut CoreSwitch,
+        snap_gen: u64,
+        helper: bool,
+    ) -> Option<LeafStop> {
+        if helper {
+            // SAFETY: see `CoreSwitch` (the running activation's handles;
+            // nothing else borrows them here).
+            let frame = unsafe { &mut *sw.cur };
+            if burst_stats::enabled() {
+                if let Some(ins) = frame.code.instructions.get(frame.pc as usize) {
+                    burst_stats::note_helper(ins.op);
+                    let recv = frame.stack.last().map_or("-", |v| v.type_name());
+                    burst_stats::note_site(&frame.code.qualname, frame.pc, ins.op, recv);
+                }
+            }
+            // A cached property site skips the helper's resolution: a
+            // pure leaf getter is evaluated in place, any other runs as an
+            // inline activation.
+            let pc = frame.pc as usize;
+            if frame
+                .code
+                .instructions
+                .get(pc)
+                .is_some_and(|i| i.op == OpCode::LoadAttr)
+            {
+                if let Some((getter, code)) = Self::cached_property_getter(frame, pc) {
+                    // SAFETY: as above.
+                    let last_pc = unsafe { &mut *sw.last };
+                    if self.core_leaf_property(frame, last_pc, pc, &getter, &code) {
                         // SAFETY: the running thread's own flag.
-                        CoreAttr::Done if unsafe { (*sw.maybe_dead).get() } => {
-                            return LeafStop::Marked;
-                        }
-                        CoreAttr::Done => continue,
-                        CoreAttr::Raised(e) => return LeafStop::Raised(e),
-                        CoreAttr::Full => return LeafStop::Step,
-                        CoreAttr::Decline => {
-                            match frame.code.instructions.get(frame.pc as usize).map(|i| i.op) {
-                                Some(OpCode::LoadAttr)
-                                    if self.core_property(sw) || self.core_getitem(sw, true) =>
-                                {
-                                    continue
-                                }
-                                Some(OpCode::StoreAttr) if self.core_setter(sw) => continue,
-                                Some(OpCode::CallKw | OpCode::CallEx)
-                                    if self.core_call_lane(sw) =>
-                                {
-                                    continue
-                                }
-                                Some(OpCode::Call)
-                                    if self.core_leaf_call_lane(sw) || self.core_type_lane(sw) =>
-                                {
-                                    continue
-                                }
-                                _ => {}
-                            }
+                        if unsafe { (*sw.maybe_dead).get() } {
+                            return Some(LeafStop::Marked);
                         }
+                        return None;
+                    }
+                    if self.core_property(sw) {
+                        return None;
                     }
                 }
-                CoreExit::Slow => {
-                    // SAFETY: as above.
-                    let frame = unsafe { &*sw.cur };
+            }
+            // SAFETY: as above (`core_property` declined untouched).
+            let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) };
+            match self.leaf_core_helper(frame, last_pc) {
+                // A released receiver or argument may have queued a
+                // finalizer (see `after_release!` in the core).
+                // SAFETY: the running thread's own flag.
+                CoreAttr::Done if unsafe { (*sw.maybe_dead).get() } => {
+                    return Some(LeafStop::Marked);
+                }
+                CoreAttr::Done => return None,
+                CoreAttr::Raised(e) => return Some(LeafStop::Raised(e)),
+                CoreAttr::Full => return Some(LeafStop::Step),
+                CoreAttr::Decline => {
                     match frame.code.instructions.get(frame.pc as usize).map(|i| i.op) {
-                        Some(OpCode::BinarySubscr) if self.core_getitem(sw, false) => continue,
-                        Some(OpCode::ForIter) if self.core_for_iter_lane(sw) => continue,
-                        Some(OpCode::CallKw | OpCode::CallEx) if self.core_call_lane(sw) => {
-                            continue
+                        Some(OpCode::LoadAttr)
+                            if self.core_property(sw) || self.core_getitem(sw, true) =>
+                        {
+                            return None
                         }
-                        Some(OpCode::BinaryOp | OpCode::SetUpdate) if self.core_set_op(sw) => {
-                            continue
+                        Some(OpCode::StoreAttr) if self.core_setter(sw) => return None,
+                        Some(OpCode::CallKw | OpCode::CallEx) if self.core_call_lane(sw) => {
+                            return None
                         }
-                        Some(OpCode::BinaryOp | OpCode::CompareOp)
-                            if self.core_operator_lane(sw) =>
+                        Some(OpCode::Call)
+                            if self.core_leaf_call_lane(sw) || self.core_type_lane(sw) =>
                         {
-                            continue
+                            return None
                         }
                         _ => {}
                     }
                 }
-                CoreExit::Reload => unreachable!("handled inside the core loop"),
             }
+        } else {
             // SAFETY: as above.
-            let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) };
-            match self.leaf_burst_slow(frame, snap_gen, last_pc) {
-                // SAFETY: as above.
-                LeafStop::Core if unsafe { (*sw.maybe_dead).get() } => return LeafStop::Marked,
-                LeafStop::Core => {}
-                stop => return stop,
+            let frame = unsafe { &*sw.cur };
+            match frame.code.instructions.get(frame.pc as usize).map(|i| i.op) {
+                Some(OpCode::BinarySubscr) if self.core_getitem(sw, false) => return None,
+                Some(OpCode::ForIter) if self.core_for_iter_lane(sw) => return None,
+                Some(OpCode::CallKw | OpCode::CallEx) if self.core_call_lane(sw) => return None,
+                Some(OpCode::BinaryOp | OpCode::SetUpdate) if self.core_set_op(sw) => return None,
+                Some(OpCode::BinaryOp | OpCode::CompareOp) if self.core_operator_lane(sw) => {
+                    return None
+                }
+                _ => {}
             }
         }
+        // SAFETY: as above.
+        let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) };
+        match self.leaf_burst_slow(frame, snap_gen, last_pc) {
+            // SAFETY: as above.
+            LeafStop::Core if unsafe { (*sw.maybe_dead).get() } => Some(LeafStop::Marked),
+            LeafStop::Core => None,
+            stop => Some(stop),
+        }
+    }
+
+    /// [`Self::core_offload`] from inside [`Self::leaf_core`]'s loop, which
+    /// holds the running activation at `at` with its operand stack at
+    /// `base` (capacity `cap`): synced, then run. `true` when the loop goes
+    /// on in place, at the new `at`: the same activation, its stack where
+    /// it was, nothing pending (what a reload would derive again is
+    /// unchanged). Otherwise the loop reloads, which ends the burst first
+    /// if it must (`sw.pending`).
+    ///
+    /// # Safety
+    ///
+    /// `at`, `base` and `cap` are the running activation's, as the core
+    /// loop holds them (every stack slot below `at.len` initialized).
+    #[inline(never)]
+    unsafe fn core_offload_in_loop(
+        &mut self,
+        sw: &mut CoreSwitch,
+        snap_gen: u64,
+        helper: bool,
+        base: *mut Object,
+        cap: usize,
+        at: &mut CoreAt,
+    ) -> bool {
+        let (cur, last) = (sw.cur, sw.last);
+        // SAFETY: see `CoreSwitch`; the caller's contract.
+        unsafe {
+            (*cur).stack.set_len(at.len);
+            (*cur).pc = at.pc as u32;
+            *last = at.last;
+        }
+        if let Some(stop) = self.core_offload(sw, snap_gen, helper) {
+            // (What the burst's own end would drop with it.)
+            sw.pending = Some(CoreExit::Stop(stop));
+            return false;
+        }
+        if sw.pending.is_some() || sw.cur != cur || sw.last != last {
+            return false;
+        }
+        // SAFETY: the same activation, unborrowed here.
+        let frame = unsafe { &mut *cur };
+        let pc = frame.pc as usize;
+        // (A reload would also count an entry at pc 0 as a call, and check
+        // the pc against the code.)
+        if frame.stack.as_mut_ptr() != base
+            || frame.stack.capacity() != cap
+            || pc == 0
+            || pc >= frame.code.instructions.len()
+        {
+            return false;
+        }
+        at.len = frame.stack.len();
+        at.pc = pc;
+        // SAFETY: the activation's last-pc slot.
+        at.last = unsafe { *last };
+        true
     }
 
     /// Run the core loop's out-of-line instruction at `frame.pc` (the
-    /// ones [`Self::leaf_core`] hands over as [`CoreExit::Helper`]): the
+    /// ones [`Self::leaf_core`] hands over to [`Self::core_offload`]): the
     /// fused `len(x)` and `x.attr` shapes, the global/attribute cache hits,
     /// and leaf builtin/type calls. Advances `frame.pc` past what ran;
     /// `Decline` touches nothing.
@@ -11440,6 +11525,29 @@ impl Interpreter {
                     }
                 };
             }
+            // Hand the instruction at `pc` over (out of line with `true`,
+            // to the full leaf arms with `false`) without leaving the loop:
+            // when the activation goes on in place, so does the loop, with
+            // no reload (see `core_offload_in_loop`).
+            macro_rules! offload {
+                ($helper:expr) => {{
+                    let mut at = CoreAt { len, pc, last };
+                    // SAFETY: the running activation's state, as held here.
+                    if unsafe {
+                        self.core_offload_in_loop(sw, snap_gen, $helper, base, cap, &mut at)
+                    } {
+                        len = at.len;
+                        pc = at.pc;
+                        last = at.last;
+                        // (The run may have allocated a site table.)
+                        cold_stamps = None;
+                        cold_mslots = None;
+                        continue;
+                    } else {
+                        break Some(CoreExit::Reload);
+                    }
+                }};
+            }
             // The code's native form (see `frame_jit`): the native copy of
             // this loop runs it from every pc the arms below leave it at,
             // and the arms only run the instructions it returns at.
@@ -11552,6 +11660,10 @@ impl Interpreter {
                                             // The scalar shapes copy inline, as
                                             // the single-load arm does.
                                             let ca = match a {
+                                                s @ (Object::Int(_)
+                                                | Object::Float(_)
+                                                | Object::Bool(_)
+                                                | Object::None) => scalar_copy(s),
                                                 Object::Unbound => break None,
                                                 other => clone_hot(other),
                                             };
@@ -11838,7 +11950,10 @@ impl Interpreter {
                                     break None;
                                 }
                                 len -= 1;
-                                self.release(std::mem::replace(&mut *slot, base.add(len).read()));
+                                self.release_hot(std::mem::replace(
+                                    &mut *slot,
+                                    base.add(len).read(),
+                                ));
                                 last = pc;
                                 pc += 1;
                                 after_release!();
@@ -11870,7 +11985,7 @@ impl Interpreter {
                                 if !Self::core_droppable(&*slot) {
                                     break None;
                                 }
-                                self.release(std::mem::replace(&mut *slot, Object::Unbound));
+                                self.release_hot(std::mem::replace(&mut *slot, Object::Unbound));
                                 last = pc;
                                 pc += 1;
                                 after_release!();
@@ -12063,7 +12178,13 @@ impl Interpreter {
                         // next instruction stores it back): the local lets go of
                         // the string, which then grows in place when nothing else
                         // holds it (CPython's `BINARY_OP_INPLACE_ADD_UNICODE`).
-                        if kind == BinOpKind::Add && pc + 1 < ninstrs {
+                        // (The left operand's tag first: a numeric add skips
+                        // the rest.)
+                        // SAFETY: `len >= 2`.
+                        if kind == BinOpKind::Add
+                            && matches!(unsafe { &*base.add(len - 2) }, Object::Str(_))
+                            && pc + 1 < ninstrs
+                        {
                             // SAFETY: `pc + 1 < ninstrs`.
                             let next = unsafe { *instrs.add(pc + 1) };
                             let li = next.arg as usize;
@@ -13826,15 +13947,15 @@ impl Interpreter {
                                 // SAFETY: `len >= argc + 2`: the self slot.
                                 let recv = unsafe { &*base.add(len - argc - 1) };
                                 let Object::List(l) = recv else {
-                                    break Some(CoreExit::Helper);
+                                    offload!(true);
                                 };
                                 if !Self::core_droppable(recv) {
-                                    break Some(CoreExit::Helper);
+                                    offload!(true);
                                 }
                                 // SAFETY: nothing below runs code while the
                                 // items are borrowed (see `GilCell::peek_mut`).
                                 let Some(items) = (unsafe { l.peek_mut() }) else {
-                                    break Some(CoreExit::Helper);
+                                    offload!(true);
                                 };
                                 let result = match (kind, argc) {
                                     (Some(LeafKind::ListAppend), 1) => {
@@ -13847,9 +13968,9 @@ impl Interpreter {
                                         Some(v) => v,
                                         // `pop from empty list` raises: the
                                         // helper's path.
-                                        None => break Some(CoreExit::Helper),
+                                        None => offload!(true),
                                     },
-                                    _ => break Some(CoreExit::Helper),
+                                    _ => offload!(true),
                                 };
                                 // SAFETY: the list (droppable) and the builtin
                                 // (count above one) leave by plain decrements;
@@ -13878,12 +13999,12 @@ impl Interpreter {
                                         == Some(LeafKind::ListAppend)) =>
                             {
                                 let Object::List(l) = &bm.receiver else {
-                                    break Some(CoreExit::Helper);
+                                    offload!(true);
                                 };
                                 // SAFETY: nothing below runs code while the
                                 // items are borrowed (see `GilCell::peek_mut`).
                                 let Some(items) = (unsafe { l.peek_mut() }) else {
-                                    break Some(CoreExit::Helper);
+                                    offload!(true);
                                 };
                                 // SAFETY: the operand moves into the list; the
                                 // empty self slot and the bound method (held
@@ -13927,7 +14048,7 @@ impl Interpreter {
                                     std::slice::from_raw_parts(base.add(first), len - first)
                                 };
                                 if !Self::core_all_droppable(ops) {
-                                    break Some(CoreExit::Helper);
+                                    offload!(true);
                                 }
                                 let r = match kind {
                                     Some(LeafKind::Fast(f)) => f(ops),
@@ -13942,7 +14063,7 @@ impl Interpreter {
                                 };
                                 match r {
                                     // A fast half declined, untouched.
-                                    None => break Some(CoreExit::Helper),
+                                    None => offload!(true),
                                     Some(Ok(v)) => {
                                         // SAFETY: every operand (checked) and
                                         // the builtin (count above one) leave
@@ -13986,7 +14107,7 @@ impl Interpreter {
                                     if matches!(&bm.function, Object::Builtin(b)
                                         if slot.get_leaf(b).is_some())
                                     {
-                                        break Some(CoreExit::Helper);
+                                        offload!(true);
                                     }
                                 }
                                 break None;
@@ -13994,11 +14115,11 @@ impl Interpreter {
                             // A leaf builtin or directly constructible type, or
                             // a bound builtin method held in a local (`append =
                             // y.append`): out of line.
-                            Object::Builtin(_) | Object::Type(_) => break Some(CoreExit::Helper),
+                            Object::Builtin(_) | Object::Type(_) => offload!(true),
                             Object::BoundMethod(bm)
                                 if matches!(&bm.function, Object::Builtin(_)) =>
                             {
-                                break Some(CoreExit::Helper)
+                                offload!(true)
                             }
                             _ => break None,
                         }
@@ -14008,7 +14129,7 @@ impl Interpreter {
                     OpCode::LoadGlobal => {
                         use weavepy_compiler::InlineCache as IC;
                         let Some(slot) = stamps!(cold_stamps, frame, ext).get(pc) else {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         };
                         if len == cap {
                             break None;
@@ -14094,10 +14215,10 @@ impl Interpreter {
                                 }
                                 hit
                             }
-                            _ => break Some(CoreExit::Helper),
+                            _ => offload!(true),
                         };
                         let Some((_, v)) = hit else {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         };
                         let v = match v {
                             s @ (Object::Int(_)
@@ -14209,14 +14330,14 @@ impl Interpreter {
                             break None;
                         }
                         let Some(cell) = frame.cells.get(ins.arg as usize) else {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         };
                         // SAFETY: a read between two instructions.
                         let Some(v) = (unsafe { cell.peek() }) else {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         };
                         if matches!(v, Object::Unbound) {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         let c = Self::clone_operand(v);
                         // SAFETY: `len < cap`.
@@ -14235,7 +14356,7 @@ impl Interpreter {
                         // loop's initialized stack.
                         match unsafe { Self::core_super_attr(code, ins, pc, base, len) } {
                             Some(n) => len = n,
-                            None => break Some(CoreExit::Helper),
+                            None => offload!(true),
                         }
                         last = pc;
                         pc += 1;
@@ -14247,12 +14368,12 @@ impl Interpreter {
                     // the site.
                     OpCode::LoadSpecial => {
                         if len == 0 || len == cap {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         // SAFETY: `len > 0`.
                         let top = unsafe { base.add(len - 1) };
                         let Object::Instance(inst) = (unsafe { &*top }) else {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         };
                         let ver = inst.cls_raw().attr_version.get();
                         let ms = mslots!(cold_mslots, ext).get(pc);
@@ -14261,7 +14382,7 @@ impl Interpreter {
                                 .map(Object::Function)
                                 .or_else(|| ms.get_inst_builtin(ver).map(Object::Builtin))
                         }) else {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         };
                         // SAFETY: `len < cap`; the instance moves up into
                         // the self slot.
@@ -14280,7 +14401,7 @@ impl Interpreter {
                     // a class receiver leaves an empty self slot.
                     OpCode::LoadMethodAttr => {
                         if len == 0 || len == cap {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         // SAFETY: `len > 0`.
                         let top = unsafe { base.add(len - 1) };
@@ -14300,7 +14421,7 @@ impl Interpreter {
                                 pc += 1;
                                 continue;
                             }
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         // `module.func(...)`: the attribute off the site's
                         // cached index (no method binds), with an empty self
@@ -14308,7 +14429,7 @@ impl Interpreter {
                         // SAFETY: as above.
                         if let Object::Module(m) = unsafe { &*top } {
                             let Some(v) = Self::core_module_attr(code, m, pc, ins.arg) else {
-                                break Some(CoreExit::Helper);
+                                offload!(true);
                             };
                             // SAFETY: `len < cap`: `top` and the slot above.
                             unsafe {
@@ -14328,7 +14449,7 @@ impl Interpreter {
                         // SAFETY: as above.
                         if let Object::Instance(inst) = unsafe { &*top } {
                             if !Self::default_getattribute(inst.cls_raw()) {
-                                break Some(CoreExit::Helper);
+                                offload!(true);
                             }
                             if inst_may_shadow(inst, code, ins.arg) {
                                 // SAFETY: `len < cap`: `top` and the slot
@@ -14339,7 +14460,7 @@ impl Interpreter {
                                     pc += 1;
                                     continue;
                                 }
-                                break Some(CoreExit::Helper);
+                                offload!(true);
                             }
                         }
                         let Some(ms) = mslots!(cold_mslots, ext).get(pc) else {
@@ -14354,7 +14475,7 @@ impl Interpreter {
                                 pc += 1;
                                 continue;
                             }
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         };
                         let f = match unsafe { &*top } {
                             Object::Instance(inst) => {
@@ -14400,7 +14521,7 @@ impl Interpreter {
                                                 pc += 1;
                                                 continue;
                                             }
-                                            None => break Some(CoreExit::Helper),
+                                            None => offload!(true),
                                         },
                                     },
                                 };
@@ -14420,7 +14541,7 @@ impl Interpreter {
                             Object::Type(cls) if Rc::strong_count(cls) > 1 => {
                                 match ms.get_held_unbound(cls.attr_version.get()) {
                                     Some(f) => f,
-                                    None => break Some(CoreExit::Helper),
+                                    None => offload!(true),
                                 }
                             }
                             // A builtin receiver's site-cached method (the
@@ -14430,7 +14551,7 @@ impl Interpreter {
                                 let Some(b) =
                                     builtin_recv_tag(recv).and_then(|tag| ms.get_builtin(tag))
                                 else {
-                                    break Some(CoreExit::Helper);
+                                    offload!(true);
                                 };
                                 // SAFETY: `len < cap`; the receiver moves up.
                                 unsafe {
@@ -14461,7 +14582,7 @@ impl Interpreter {
                     OpCode::LoadAttr => {
                         use weavepy_compiler::InlineCache as IC;
                         if len == 0 {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         // SAFETY: `len > 0`.
                         let top = unsafe { base.add(len - 1) };
@@ -14492,7 +14613,7 @@ impl Interpreter {
                                             pc += 1;
                                             continue;
                                         }
-                                        break Some(CoreExit::Helper);
+                                        offload!(true);
                                     }
                                 }
                             }
@@ -14507,7 +14628,7 @@ impl Interpreter {
                                         pc += 1;
                                         continue;
                                     }
-                                    _ => break Some(CoreExit::Helper),
+                                    _ => offload!(true),
                                 }
                             }
                             // A function's stored attribute (`wrapper.calls`).
@@ -14519,7 +14640,7 @@ impl Interpreter {
                                     after_release!();
                                     continue;
                                 }
-                                break Some(CoreExit::Helper);
+                                offload!(true);
                             }
                             // A builtin container's method read as a value
                             // (`append = y.append`), a frame's or code
@@ -14533,7 +14654,7 @@ impl Interpreter {
                                     after_release!();
                                     continue;
                                 }
-                                break Some(CoreExit::Helper);
+                                offload!(true);
                             }
                         };
                         // A natively served instance's public field (see
@@ -14548,7 +14669,7 @@ impl Interpreter {
                                     pc += 1;
                                     continue;
                                 }
-                                _ => break Some(CoreExit::Helper),
+                                _ => offload!(true),
                             }
                         }
                         // SAFETY: a read between two instructions (see
@@ -14576,7 +14697,7 @@ impl Interpreter {
                                     pc += 1;
                                     continue;
                                 }
-                                break Some(CoreExit::Helper);
+                                offload!(true);
                             }
                         };
                         let cls = inst.cls_raw();
@@ -14584,16 +14705,16 @@ impl Interpreter {
                             || cls.native_kind.get() != 0
                             || !Self::core_droppable(unsafe { &*top })
                         {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         // SAFETY: a read between two instructions (see
                         // `GilCell::peek`).
                         let Some((k, v)) = (unsafe { inst.attr_peek_index(key_idx as usize) })
                         else {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         };
                         if !slot_name_matches(code, ins.arg, k) {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         if let Some(e) = ext {
                             field_slot_note(e, code, pc, inst, key_idx);
@@ -14608,7 +14729,7 @@ impl Interpreter {
                     // droppable one (the helper's `leaf_store_attr` first case).
                     OpCode::StoreAttr => {
                         if len < 2 {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         // SAFETY: `len >= 2`.
                         let (recv, val) = unsafe { (&*base.add(len - 1), &*base.add(len - 2)) };
@@ -14627,14 +14748,14 @@ impl Interpreter {
                                     after_release!();
                                     continue;
                                 }
-                                break Some(CoreExit::Helper);
+                                offload!(true);
                             }
-                            _ => break Some(CoreExit::Helper),
+                            _ => offload!(true),
                         };
                         if !Self::core_droppable(recv)
                             || !Self::core_store_attr(code, inst, pc, ins.arg, val)
                         {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         // SAFETY: the value moved into the dict (its slot is a
                         // stand-in now) and the receiver leaves the stack by a
@@ -14678,7 +14799,7 @@ impl Interpreter {
                                 _ => false,
                             };
                         if !ok {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         last = pc;
                         pc += 1;
@@ -14722,7 +14843,7 @@ impl Interpreter {
                                             continue;
                                         }
                                     }
-                                    break Some(CoreExit::Helper);
+                                    offload!(true);
                                 }
                             }
                             break Some(CoreExit::Stop(LeafStop::Step));
@@ -14906,7 +15027,7 @@ impl Interpreter {
                     // arm), when the iterable leaves by a plain decrement.
                     OpCode::GetIter => {
                         if len == 0 {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         // SAFETY: `len > 0`.
                         let top = unsafe { base.add(len - 1) };
@@ -14932,10 +15053,10 @@ impl Interpreter {
                                 | Object::Str(_)
                         ) || !Self::core_droppable(v)
                         {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         }
                         let Ok(it) = v.make_iter() else {
-                            break Some(CoreExit::Helper);
+                            offload!(true);
                         };
                         let it = Object::Iter(Rc::new(RefCell::new(it)));
                         // SAFETY: the iterable (droppable) is replaced in place.
@@ -14951,7 +15072,7 @@ impl Interpreter {
                     op if !SLOW_LEAF_OPS[op as u8 as usize] => {
                         break Some(CoreExit::Stop(LeafStop::Step))
                     }
-                    _ => break None,
+                    _ => offload!(false),
                 }
             };
             if matches!(stop, Some(CoreExit::Reload)) {
@@ -19060,9 +19181,9 @@ impl Interpreter {
     /// decrement (see [`Self::core_droppable`]).
     #[inline(always)]
     fn core_drops_plain() -> bool {
-        !crate::capi_watchers::dicts_active()
-            && !crate::capi_watchers::funcs_active()
-            && !crate::stdlib::testinternalcapi_mod::reftrace_print_active()
+        // (The dict and function watchers' and the reftrace printer's
+        // flags, tested with one load.)
+        crate::sync::release_gates() & crate::sync::RELEASE_OBSERVER_GATES == 0
     }
 
     /// [`Self::core_droppable`] for every one of `ops`, with the global
@@ -67497,10 +67618,8 @@ enum DictStep {
 enum CoreExit {
     /// End the burst.
     Stop(LeafStop),
-    /// The instruction at `frame.pc` runs out of line
-    /// (`leaf_core_helper`).
-    Helper,
-    /// The instruction at `frame.pc` needs the full leaf arms.
+    /// The instruction at `frame.pc` needs the full leaf arms (see
+    /// [`Interpreter::core_offload`]).
     Slow,
     /// (Inside [`Interpreter::leaf_core`] only.) An in-loop call or
     /// return synced and moved the running activation: reload.
@@ -67512,6 +67631,14 @@ enum CoreExit {
     Switch,
 }
 
+/// The core loop's view of its running activation as it hands an
+/// instruction over in place (see [`Interpreter::core_offload_in_loop`]).
+struct CoreAt {
+    len: usize,
+    pc: usize,
+    last: usize,
+}
+
 /// How [`Interpreter::leaf_core_attr`] ran its instruction.
 enum CoreAttr {
     /// Executed.
@@ -73718,8 +73845,15 @@ fn scalar_copy(v: &Object) -> Object {
         v,
         Object::None | Object::Unbound | Object::Bool(_) | Object::Int(_) | Object::Float(_)
     ));
-    // SAFETY: a scalar owns nothing: its bits are a complete copy.
-    unsafe { std::ptr::read(v) }
+    // SAFETY: a scalar owns nothing: its bits are a complete copy. (Read
+    // as opaque bytes: typed as an `Object` known to be a scalar, the copy
+    // is split into the variants' fields, a byte, a word and padding moved
+    // apart through the stack.)
+    unsafe {
+        std::mem::transmute::<std::mem::MaybeUninit<[u64; 2]>, Object>(std::ptr::read(
+            std::ptr::from_ref(v).cast::<std::mem::MaybeUninit<[u64; 2]>>(),
+        ))
+    }
 }
 
 /// `Object::Int(v)` built as two whole words (see [`clone_hot`]): the
diff --git a/crates/weavepy-vm/src/sync.rs b/crates/weavepy-vm/src/sync.rs
index e45553fd..6fc89b23 100644
--- a/crates/weavepy-vm/src/sync.rs
+++ b/crates/weavepy-vm/src/sync.rs
@@ -434,6 +434,23 @@ pub(crate) static RELEASE_GATES: ReleaseGates = ReleaseGates {
     reftrace_print: std::sync::atomic::AtomicBool::new(false),
 };
 
+/// The four [`RELEASE_GATES`] flags read with one 32-bit load, as compiled
+/// code reads them (see [`crate::plain_release_gate`]): zero when a shared
+/// value's release is a plain decrement.
+#[inline(always)]
+pub(crate) fn release_gates() -> u32 {
+    // SAFETY: `ReleaseGates` is four `AtomicBool`s, 4-aligned (`repr(C)`):
+    // its bytes are a valid `AtomicU32`, read (relaxed) as one word.
+    unsafe {
+        (*std::ptr::from_ref(&RELEASE_GATES).cast::<std::sync::atomic::AtomicU32>())
+            .load(Ordering::Relaxed)
+    }
+}
+
+/// The [`release_gates`] bits of the release observers alone (every flag
+/// but the reference-count bias's `rc_shared`).
+pub(crate) const RELEASE_OBSERVER_GATES: u32 = u32::from_ne_bytes([0, 0xff, 0xff, 0xff]);
+
 /// Whether reference counts must be updated atomically: set whenever the
 /// cell bias is revoked, and also before spawning a thread that may touch
 /// objects before it registers (see [`crate::rc`]). Never cleared.

From a18e8da1a7d539b4ffcd7fd91747664ecc88b990 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 23:33:50 -0700
Subject: [PATCH 126/173] perf: start bursts in the right core-loop copy and
 run scalar unary ops in it

- A burst used to start in the interpreted copy of the core loop and
  switch to the native copy when its code had native form, paying two
  reload prologues. leaf_burst now starts in the copy the running code
  wants, and the prologue tests for a copy mismatch right after deriving
  the locals, before the rest of what the other copy derives again.
- UNARY_OP of a scalar (`not` of a bool, int or None, `-` and `~` of an
  int) has a core arm; it used to hand off to the full leaf arms.
- Handoffs to the full leaf arms from the core loop's default arm skip
  the lane checks that only apply to opcodes with arms of their own, and
  the offload body is inlined into its in-loop caller.

Adds tests/regrtest/test_core_loop_handoffs.py (unary shapes, in-place
handoffs with finalizer order, a raise from a helper's traceback line,
line events).

Measured (quick build, instructions retired, PYTHONHASHSEED=0):
- suite geomean 0.9964x of the previous commit (scimark 0.987x,
  closures_decorators 0.991x, graph_search 0.991x, exceptions 0.992x,
  stdlib_utils 0.992x).
- frame-JIT-off probe `x = not a`: 455 -> 171 instructions per statement.
---
 crates/weavepy-vm/src/lib.rs              | 224 ++++++++++++++--------
 tests/regrtest/test_core_loop_handoffs.py | 183 ++++++++++++++++++
 2 files changed, 324 insertions(+), 83 deletions(-)
 create mode 100644 tests/regrtest/test_core_loop_handoffs.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 0466da01..cc49ffa8 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -10867,9 +10867,21 @@ impl Interpreter {
     #[inline(never)]
     fn leaf_burst(&mut self, sw: &mut CoreSwitch, snap_gen: u64) -> LeafStop {
         // Which copy of the core loop runs: the native one drives the
-        // running code's native form (see `frame_jit`).
+        // running code's native form (see `frame_jit`). A burst starts in
+        // the copy its activation's code wants (the other would only switch
+        // over).
         #[cfg(feature = "jit")]
-        let mut native = false;
+        let mut native = {
+            // SAFETY: see `CoreSwitch` (the running activation's handles;
+            // nothing else borrows them here).
+            let frame = unsafe { &*sw.cur };
+            code_vm_ext(&frame.code).is_some_and(|e| {
+                // SAFETY: as in `leaf_core` (a GIL-serialized read of the
+                // locals vector's length).
+                let nlocals = unsafe { (*frame.locals.as_ptr()).len() };
+                e.frame_jit.get(nlocals).is_some()
+            })
+        };
         loop {
             #[cfg(feature = "jit")]
             let exit = if native {
@@ -10884,7 +10896,7 @@ impl Interpreter {
                 CoreExit::Switch => native = !native,
                 CoreExit::Stop(stop) => return stop,
                 CoreExit::Slow => {
-                    if let Some(stop) = self.core_offload(sw, snap_gen, false) {
+                    if let Some(stop) = self.core_offload(sw, snap_gen) {
                         return stop;
                     }
                 }
@@ -10893,19 +10905,27 @@ impl Interpreter {
         }
     }
 
-    /// Run the instruction the core loop handed over at `sw`'s (synced)
-    /// running activation's pc: out of line (`helper`: the fused shapes,
-    /// cache hits and leaf calls of [`Self::leaf_core_helper`] and the
-    /// lanes) or with the full leaf arms. `None` when the core loop goes on
-    /// (on the activation `sw` now runs), else how the burst ends.
+    /// [`Self::core_offload_run`] of a [`CoreExit::Slow`] handoff.
     #[inline(never)]
-    fn core_offload(
+    fn core_offload(&mut self, sw: &mut CoreSwitch, snap_gen: u64) -> Option<LeafStop> {
+        self.core_offload_run(sw, snap_gen, OFFLOAD_SLOW)
+    }
+
+    /// Run the instruction the core loop handed over at `sw`'s (synced)
+    /// running activation's pc, as `mode` says: [`OFFLOAD_HELPER`] out of
+    /// line (the fused shapes, cache hits and leaf calls of
+    /// [`Self::leaf_core_helper`] and the lanes), [`OFFLOAD_SLOW`] on a
+    /// lane or with the full leaf arms, [`OFFLOAD_ARMS`] with the full leaf
+    /// arms. `None` when the core loop goes on (on the activation `sw` now
+    /// runs), else how the burst ends.
+    #[inline(always)]
+    fn core_offload_run(
         &mut self,
         sw: &mut CoreSwitch,
         snap_gen: u64,
-        helper: bool,
+        mode: u8,
     ) -> Option<LeafStop> {
-        if helper {
+        if mode == OFFLOAD_HELPER {
             // SAFETY: see `CoreSwitch` (the running activation's handles;
             // nothing else borrows them here).
             let frame = unsafe { &mut *sw.cur };
@@ -10973,7 +10993,7 @@ impl Interpreter {
                     }
                 }
             }
-        } else {
+        } else if mode == OFFLOAD_SLOW {
             // SAFETY: as above.
             let frame = unsafe { &*sw.cur };
             match frame.code.instructions.get(frame.pc as usize).map(|i| i.op) {
@@ -10997,7 +11017,7 @@ impl Interpreter {
         }
     }
 
-    /// [`Self::core_offload`] from inside [`Self::leaf_core`]'s loop, which
+    /// [`Self::core_offload_run`] from inside [`Self::leaf_core`]'s loop, which
     /// holds the running activation at `at` with its operand stack at
     /// `base` (capacity `cap`): synced, then run. `true` when the loop goes
     /// on in place, at the new `at`: the same activation, its stack where
@@ -11010,11 +11030,10 @@ impl Interpreter {
     /// `at`, `base` and `cap` are the running activation's, as the core
     /// loop holds them (every stack slot below `at.len` initialized).
     #[inline(never)]
-    unsafe fn core_offload_in_loop(
+    unsafe fn core_offload_in_loop<const MODE: u8>(
         &mut self,
         sw: &mut CoreSwitch,
         snap_gen: u64,
-        helper: bool,
         base: *mut Object,
         cap: usize,
         at: &mut CoreAt,
@@ -11026,7 +11045,7 @@ impl Interpreter {
             (*cur).pc = at.pc as u32;
             *last = at.last;
         }
-        if let Some(stop) = self.core_offload(sw, snap_gen, helper) {
+        if let Some(stop) = self.core_offload_run(sw, snap_gen, MODE) {
             // (What the burst's own end would drop with it.)
             sw.pending = Some(CoreExit::Stop(stop));
             return false;
@@ -11485,6 +11504,18 @@ impl Interpreter {
             // during an activation, and nothing here runs Python code).
             let locals: &mut Vec<Object> = unsafe { &mut *frame.locals.as_ptr() };
             let (lbase, nlocals) = (locals.as_mut_ptr(), locals.len());
+            // The code's native form (see `frame_jit`): the native copy of
+            // this loop runs it from every pc the arms below leave it at,
+            // and the arms only run the instructions it returns at. (Tested
+            // before the rest of the prologue, which the other copy derives
+            // again.)
+            #[cfg(feature = "jit")]
+            let native = ext.and_then(|e| e.frame_jit.get(nlocals));
+            #[cfg(feature = "jit")]
+            if NATIVE != native.is_some() {
+                // (The activation is synced at every reload.)
+                return CoreExit::Switch;
+            }
             // (The namespaces are read off the frame at their uses: holding
             // them here costs every switch a spill.)
             // The arms' colder per-activation state (the global and class
@@ -11525,16 +11556,16 @@ impl Interpreter {
                     }
                 };
             }
-            // Hand the instruction at `pc` over (out of line with `true`,
-            // to the full leaf arms with `false`) without leaving the loop:
-            // when the activation goes on in place, so does the loop, with
-            // no reload (see `core_offload_in_loop`).
+            // Hand the instruction at `pc` over (see `core_offload_run`'s
+            // modes) without leaving the loop: when the activation goes on
+            // in place, so does the loop, with no reload (see
+            // `core_offload_in_loop`).
             macro_rules! offload {
-                ($helper:expr) => {{
+                ($mode:expr) => {{
                     let mut at = CoreAt { len, pc, last };
                     // SAFETY: the running activation's state, as held here.
                     if unsafe {
-                        self.core_offload_in_loop(sw, snap_gen, $helper, base, cap, &mut at)
+                        self.core_offload_in_loop::<{ $mode }>(sw, snap_gen, base, cap, &mut at)
                     } {
                         len = at.len;
                         pc = at.pc;
@@ -11548,16 +11579,6 @@ impl Interpreter {
                     }
                 }};
             }
-            // The code's native form (see `frame_jit`): the native copy of
-            // this loop runs it from every pc the arms below leave it at,
-            // and the arms only run the instructions it returns at.
-            #[cfg(feature = "jit")]
-            let native = ext.and_then(|e| e.frame_jit.get(nlocals));
-            #[cfg(feature = "jit")]
-            if NATIVE != native.is_some() {
-                // (The activation is synced at every reload.)
-                return CoreExit::Switch;
-            }
             // The native code writes the stack at fixed offsets: a stack
             // that started small (a materialized frame's) grows first.
             #[cfg(feature = "jit")]
@@ -12143,6 +12164,35 @@ impl Interpreter {
                         last = pc;
                         pc += 1;
                     }
+                    // `not x`, `-x` and `~x` of a scalar, replaced in place
+                    // (nothing to release); any other shape is the full
+                    // leaf arm's.
+                    OpCode::UnaryOp => {
+                        if len == 0 {
+                            break None;
+                        }
+                        // SAFETY: `len > 0`.
+                        let top = unsafe { base.add(len - 1) };
+                        // SAFETY: as in the full handler (the compiler only
+                        // emits valid kinds).
+                        let kind: UnaryKind = unsafe { std::mem::transmute(ins.arg as u8) };
+                        // SAFETY: as above.
+                        let r = match (unsafe { &*top }, kind) {
+                            (Object::Bool(b), UnaryKind::Not) => bool_hot(!*b),
+                            (Object::Int(i), UnaryKind::Not) => bool_hot(*i == 0),
+                            (Object::None, UnaryKind::Not) => bool_hot(true),
+                            (Object::Int(i), UnaryKind::Neg) => match i.checked_neg() {
+                                Some(r) => int_hot(r),
+                                None => break None,
+                            },
+                            (Object::Int(i), UnaryKind::Invert) => int_hot(!*i),
+                            _ => break None,
+                        };
+                        // SAFETY: a scalar operand is overwritten in place.
+                        unsafe { top.write(r) };
+                        last = pc;
+                        pc += 1;
+                    }
                     OpCode::PopJumpIfNone | OpCode::PopJumpIfNotNone => {
                         if len == 0 {
                             break None;
@@ -13947,15 +13997,15 @@ impl Interpreter {
                                 // SAFETY: `len >= argc + 2`: the self slot.
                                 let recv = unsafe { &*base.add(len - argc - 1) };
                                 let Object::List(l) = recv else {
-                                    offload!(true);
+                                    offload!(OFFLOAD_HELPER);
                                 };
                                 if !Self::core_droppable(recv) {
-                                    offload!(true);
+                                    offload!(OFFLOAD_HELPER);
                                 }
                                 // SAFETY: nothing below runs code while the
                                 // items are borrowed (see `GilCell::peek_mut`).
                                 let Some(items) = (unsafe { l.peek_mut() }) else {
-                                    offload!(true);
+                                    offload!(OFFLOAD_HELPER);
                                 };
                                 let result = match (kind, argc) {
                                     (Some(LeafKind::ListAppend), 1) => {
@@ -13968,9 +14018,9 @@ impl Interpreter {
                                         Some(v) => v,
                                         // `pop from empty list` raises: the
                                         // helper's path.
-                                        None => offload!(true),
+                                        None => offload!(OFFLOAD_HELPER),
                                     },
-                                    _ => offload!(true),
+                                    _ => offload!(OFFLOAD_HELPER),
                                 };
                                 // SAFETY: the list (droppable) and the builtin
                                 // (count above one) leave by plain decrements;
@@ -13999,12 +14049,12 @@ impl Interpreter {
                                         == Some(LeafKind::ListAppend)) =>
                             {
                                 let Object::List(l) = &bm.receiver else {
-                                    offload!(true);
+                                    offload!(OFFLOAD_HELPER);
                                 };
                                 // SAFETY: nothing below runs code while the
                                 // items are borrowed (see `GilCell::peek_mut`).
                                 let Some(items) = (unsafe { l.peek_mut() }) else {
-                                    offload!(true);
+                                    offload!(OFFLOAD_HELPER);
                                 };
                                 // SAFETY: the operand moves into the list; the
                                 // empty self slot and the bound method (held
@@ -14048,7 +14098,7 @@ impl Interpreter {
                                     std::slice::from_raw_parts(base.add(first), len - first)
                                 };
                                 if !Self::core_all_droppable(ops) {
-                                    offload!(true);
+                                    offload!(OFFLOAD_HELPER);
                                 }
                                 let r = match kind {
                                     Some(LeafKind::Fast(f)) => f(ops),
@@ -14063,7 +14113,7 @@ impl Interpreter {
                                 };
                                 match r {
                                     // A fast half declined, untouched.
-                                    None => offload!(true),
+                                    None => offload!(OFFLOAD_HELPER),
                                     Some(Ok(v)) => {
                                         // SAFETY: every operand (checked) and
                                         // the builtin (count above one) leave
@@ -14107,7 +14157,7 @@ impl Interpreter {
                                     if matches!(&bm.function, Object::Builtin(b)
                                         if slot.get_leaf(b).is_some())
                                     {
-                                        offload!(true);
+                                        offload!(OFFLOAD_HELPER);
                                     }
                                 }
                                 break None;
@@ -14115,11 +14165,11 @@ impl Interpreter {
                             // A leaf builtin or directly constructible type, or
                             // a bound builtin method held in a local (`append =
                             // y.append`): out of line.
-                            Object::Builtin(_) | Object::Type(_) => offload!(true),
+                            Object::Builtin(_) | Object::Type(_) => offload!(OFFLOAD_HELPER),
                             Object::BoundMethod(bm)
                                 if matches!(&bm.function, Object::Builtin(_)) =>
                             {
-                                offload!(true)
+                                offload!(OFFLOAD_HELPER)
                             }
                             _ => break None,
                         }
@@ -14129,7 +14179,7 @@ impl Interpreter {
                     OpCode::LoadGlobal => {
                         use weavepy_compiler::InlineCache as IC;
                         let Some(slot) = stamps!(cold_stamps, frame, ext).get(pc) else {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         };
                         if len == cap {
                             break None;
@@ -14215,10 +14265,10 @@ impl Interpreter {
                                 }
                                 hit
                             }
-                            _ => offload!(true),
+                            _ => offload!(OFFLOAD_HELPER),
                         };
                         let Some((_, v)) = hit else {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         };
                         let v = match v {
                             s @ (Object::Int(_)
@@ -14330,14 +14380,14 @@ impl Interpreter {
                             break None;
                         }
                         let Some(cell) = frame.cells.get(ins.arg as usize) else {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         };
                         // SAFETY: a read between two instructions.
                         let Some(v) = (unsafe { cell.peek() }) else {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         };
                         if matches!(v, Object::Unbound) {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         let c = Self::clone_operand(v);
                         // SAFETY: `len < cap`.
@@ -14356,7 +14406,7 @@ impl Interpreter {
                         // loop's initialized stack.
                         match unsafe { Self::core_super_attr(code, ins, pc, base, len) } {
                             Some(n) => len = n,
-                            None => offload!(true),
+                            None => offload!(OFFLOAD_HELPER),
                         }
                         last = pc;
                         pc += 1;
@@ -14368,12 +14418,12 @@ impl Interpreter {
                     // the site.
                     OpCode::LoadSpecial => {
                         if len == 0 || len == cap {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         // SAFETY: `len > 0`.
                         let top = unsafe { base.add(len - 1) };
                         let Object::Instance(inst) = (unsafe { &*top }) else {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         };
                         let ver = inst.cls_raw().attr_version.get();
                         let ms = mslots!(cold_mslots, ext).get(pc);
@@ -14382,7 +14432,7 @@ impl Interpreter {
                                 .map(Object::Function)
                                 .or_else(|| ms.get_inst_builtin(ver).map(Object::Builtin))
                         }) else {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         };
                         // SAFETY: `len < cap`; the instance moves up into
                         // the self slot.
@@ -14401,7 +14451,7 @@ impl Interpreter {
                     // a class receiver leaves an empty self slot.
                     OpCode::LoadMethodAttr => {
                         if len == 0 || len == cap {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         // SAFETY: `len > 0`.
                         let top = unsafe { base.add(len - 1) };
@@ -14421,7 +14471,7 @@ impl Interpreter {
                                 pc += 1;
                                 continue;
                             }
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         // `module.func(...)`: the attribute off the site's
                         // cached index (no method binds), with an empty self
@@ -14429,7 +14479,7 @@ impl Interpreter {
                         // SAFETY: as above.
                         if let Object::Module(m) = unsafe { &*top } {
                             let Some(v) = Self::core_module_attr(code, m, pc, ins.arg) else {
-                                offload!(true);
+                                offload!(OFFLOAD_HELPER);
                             };
                             // SAFETY: `len < cap`: `top` and the slot above.
                             unsafe {
@@ -14449,7 +14499,7 @@ impl Interpreter {
                         // SAFETY: as above.
                         if let Object::Instance(inst) = unsafe { &*top } {
                             if !Self::default_getattribute(inst.cls_raw()) {
-                                offload!(true);
+                                offload!(OFFLOAD_HELPER);
                             }
                             if inst_may_shadow(inst, code, ins.arg) {
                                 // SAFETY: `len < cap`: `top` and the slot
@@ -14460,7 +14510,7 @@ impl Interpreter {
                                     pc += 1;
                                     continue;
                                 }
-                                offload!(true);
+                                offload!(OFFLOAD_HELPER);
                             }
                         }
                         let Some(ms) = mslots!(cold_mslots, ext).get(pc) else {
@@ -14475,7 +14525,7 @@ impl Interpreter {
                                 pc += 1;
                                 continue;
                             }
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         };
                         let f = match unsafe { &*top } {
                             Object::Instance(inst) => {
@@ -14521,7 +14571,7 @@ impl Interpreter {
                                                 pc += 1;
                                                 continue;
                                             }
-                                            None => offload!(true),
+                                            None => offload!(OFFLOAD_HELPER),
                                         },
                                     },
                                 };
@@ -14541,7 +14591,7 @@ impl Interpreter {
                             Object::Type(cls) if Rc::strong_count(cls) > 1 => {
                                 match ms.get_held_unbound(cls.attr_version.get()) {
                                     Some(f) => f,
-                                    None => offload!(true),
+                                    None => offload!(OFFLOAD_HELPER),
                                 }
                             }
                             // A builtin receiver's site-cached method (the
@@ -14551,7 +14601,7 @@ impl Interpreter {
                                 let Some(b) =
                                     builtin_recv_tag(recv).and_then(|tag| ms.get_builtin(tag))
                                 else {
-                                    offload!(true);
+                                    offload!(OFFLOAD_HELPER);
                                 };
                                 // SAFETY: `len < cap`; the receiver moves up.
                                 unsafe {
@@ -14582,7 +14632,7 @@ impl Interpreter {
                     OpCode::LoadAttr => {
                         use weavepy_compiler::InlineCache as IC;
                         if len == 0 {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         // SAFETY: `len > 0`.
                         let top = unsafe { base.add(len - 1) };
@@ -14613,7 +14663,7 @@ impl Interpreter {
                                             pc += 1;
                                             continue;
                                         }
-                                        offload!(true);
+                                        offload!(OFFLOAD_HELPER);
                                     }
                                 }
                             }
@@ -14628,7 +14678,7 @@ impl Interpreter {
                                         pc += 1;
                                         continue;
                                     }
-                                    _ => offload!(true),
+                                    _ => offload!(OFFLOAD_HELPER),
                                 }
                             }
                             // A function's stored attribute (`wrapper.calls`).
@@ -14640,7 +14690,7 @@ impl Interpreter {
                                     after_release!();
                                     continue;
                                 }
-                                offload!(true);
+                                offload!(OFFLOAD_HELPER);
                             }
                             // A builtin container's method read as a value
                             // (`append = y.append`), a frame's or code
@@ -14654,7 +14704,7 @@ impl Interpreter {
                                     after_release!();
                                     continue;
                                 }
-                                offload!(true);
+                                offload!(OFFLOAD_HELPER);
                             }
                         };
                         // A natively served instance's public field (see
@@ -14669,7 +14719,7 @@ impl Interpreter {
                                     pc += 1;
                                     continue;
                                 }
-                                _ => offload!(true),
+                                _ => offload!(OFFLOAD_HELPER),
                             }
                         }
                         // SAFETY: a read between two instructions (see
@@ -14697,7 +14747,7 @@ impl Interpreter {
                                     pc += 1;
                                     continue;
                                 }
-                                offload!(true);
+                                offload!(OFFLOAD_HELPER);
                             }
                         };
                         let cls = inst.cls_raw();
@@ -14705,16 +14755,16 @@ impl Interpreter {
                             || cls.native_kind.get() != 0
                             || !Self::core_droppable(unsafe { &*top })
                         {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         // SAFETY: a read between two instructions (see
                         // `GilCell::peek`).
                         let Some((k, v)) = (unsafe { inst.attr_peek_index(key_idx as usize) })
                         else {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         };
                         if !slot_name_matches(code, ins.arg, k) {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         if let Some(e) = ext {
                             field_slot_note(e, code, pc, inst, key_idx);
@@ -14729,7 +14779,7 @@ impl Interpreter {
                     // droppable one (the helper's `leaf_store_attr` first case).
                     OpCode::StoreAttr => {
                         if len < 2 {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         // SAFETY: `len >= 2`.
                         let (recv, val) = unsafe { (&*base.add(len - 1), &*base.add(len - 2)) };
@@ -14748,14 +14798,14 @@ impl Interpreter {
                                     after_release!();
                                     continue;
                                 }
-                                offload!(true);
+                                offload!(OFFLOAD_HELPER);
                             }
-                            _ => offload!(true),
+                            _ => offload!(OFFLOAD_HELPER),
                         };
                         if !Self::core_droppable(recv)
                             || !Self::core_store_attr(code, inst, pc, ins.arg, val)
                         {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         // SAFETY: the value moved into the dict (its slot is a
                         // stand-in now) and the receiver leaves the stack by a
@@ -14799,7 +14849,7 @@ impl Interpreter {
                                 _ => false,
                             };
                         if !ok {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         last = pc;
                         pc += 1;
@@ -14843,7 +14893,7 @@ impl Interpreter {
                                             continue;
                                         }
                                     }
-                                    offload!(true);
+                                    offload!(OFFLOAD_HELPER);
                                 }
                             }
                             break Some(CoreExit::Stop(LeafStop::Step));
@@ -15027,7 +15077,7 @@ impl Interpreter {
                     // arm), when the iterable leaves by a plain decrement.
                     OpCode::GetIter => {
                         if len == 0 {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         // SAFETY: `len > 0`.
                         let top = unsafe { base.add(len - 1) };
@@ -15053,10 +15103,10 @@ impl Interpreter {
                                 | Object::Str(_)
                         ) || !Self::core_droppable(v)
                         {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         }
                         let Ok(it) = v.make_iter() else {
-                            offload!(true);
+                            offload!(OFFLOAD_HELPER);
                         };
                         let it = Object::Iter(Rc::new(RefCell::new(it)));
                         // SAFETY: the iterable (droppable) is replaced in place.
@@ -15072,7 +15122,7 @@ impl Interpreter {
                     op if !SLOW_LEAF_OPS[op as u8 as usize] => {
                         break Some(CoreExit::Stop(LeafStop::Step))
                     }
-                    _ => offload!(false),
+                    _ => offload!(OFFLOAD_ARMS),
                 }
             };
             if matches!(stop, Some(CoreExit::Reload)) {
@@ -67631,6 +67681,13 @@ enum CoreExit {
     Switch,
 }
 
+/// [`Interpreter::core_offload_run`]'s modes: an out-of-line instruction
+/// (the helper, then its lanes), a handoff that may take a lane before the
+/// full leaf arms, and one only the full leaf arms run.
+const OFFLOAD_HELPER: u8 = 0;
+const OFFLOAD_SLOW: u8 = 1;
+const OFFLOAD_ARMS: u8 = 2;
+
 /// The core loop's view of its running activation as it hands an
 /// instruction over in place (see [`Interpreter::core_offload_in_loop`]).
 struct CoreAt {
@@ -67774,6 +67831,7 @@ static CORE_LEAF_OPS: [bool; 256] = {
         OpCode::LoadDeref,
         OpCode::BuildList,
         OpCode::ContainsOp,
+        OpCode::UnaryOp,
     ];
     let mut i = 0;
     while i < ops.len() {
diff --git a/tests/regrtest/test_core_loop_handoffs.py b/tests/regrtest/test_core_loop_handoffs.py
new file mode 100644
index 00000000..aad069c3
--- /dev/null
+++ b/tests/regrtest/test_core_loop_handoffs.py
@@ -0,0 +1,183 @@
+"""The core loop's in-place handoffs and its scalar arms keep exact semantics.
+
+An instruction the core loop runs out of line (a cache-hit helper, a lane,
+or the full leaf arms) goes on in the same loop when the activation is
+unchanged. Releases, finalizers, raises, tracebacks and line events must
+come out exactly as if every instruction ran on its own, and the unary
+arms must agree with the full handler on every operand shape.
+"""
+
+import sys
+import unittest
+import warnings
+
+
+class Slotted:
+    __slots__ = ("a",)
+
+    def __init__(self, a):
+        self.a = a
+
+
+class WithProp:
+    def __init__(self, v):
+        self._v = v
+
+    @property
+    def v(self):
+        return self._v
+
+
+class Plain:
+    k = 3
+
+
+LOG = []
+
+
+class Noisy:
+    def __init__(self, tag):
+        self.tag = tag
+
+    def __del__(self):
+        LOG.append(("del", self.tag))
+
+
+def unary_table(values):
+    out = []
+    for v in values:
+        out.append(not v)
+        try:
+            out.append(-v)
+        except TypeError:
+            out.append("neg-type")
+        try:
+            out.append(~v)
+        except TypeError:
+            out.append("inv-type")
+    return out
+
+
+def handoff_mix(n):
+    # Attribute and global reads that take the helper, `not` and
+    # comprehension-free builtins in a loop with plain local arithmetic.
+    p = WithProp(2)
+    s = Slotted(5)
+    total = 0
+    for i in range(n):
+        total += p.v + s.a + Plain.k + len("abc") + abs(-i)
+        flag = not (i & 1)
+        if flag:
+            total -= -i
+        t = (i, i + 1, i + 2, i + 3)
+        total += t[3] - t[0]
+    return total
+
+
+def release_order(n):
+    for i in range(n):
+        x = Noisy(i)
+        LOG.append(("bound", i))
+        x = None
+        LOG.append(("after", i))
+    return LOG
+
+
+def raise_in_helper(objs):
+    seen = 0
+    for o in objs:
+        seen += o.v  # line A
+    return seen
+
+
+class CoreUnaryTest(unittest.TestCase):
+    def test_unary_shapes(self):
+        values = [0, 1, -1, 7, 2**62, -(2**63), 2**63 - 1, None, True, False,
+                  0.0, -1.5, "", "x", (), (1,), [], [0]]
+        expect = []
+        for v in values:
+            expect.append(not v)
+            try:
+                expect.append(-v)
+            except TypeError:
+                expect.append("neg-type")
+            try:
+                expect.append(~v)
+            except TypeError:
+                expect.append("inv-type")
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            for _ in range(50):
+                got = unary_table(values)
+        self.assertEqual(got, expect)
+        # The overflowing negation is a long, not a wrapped int.
+        self.assertEqual(-(-(2**63)), 2**63)
+        self.assertEqual(type(not 5), bool)
+
+    def test_invert_bool_warns(self):
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always", DeprecationWarning)
+            for _ in range(3):
+                unary_table([True])
+        self.assertTrue(any(issubclass(x.category, DeprecationWarning) for x in w))
+
+
+class HandoffTest(unittest.TestCase):
+    def test_mixed_loop(self):
+        def reference(n):
+            total = 0
+            for i in range(n):
+                total += 2 + 5 + 3 + 3 + i
+                if not (i & 1):
+                    total += i
+                total += 3
+            return total
+
+        for n in (0, 1, 7, 500, 3000):
+            self.assertEqual(handoff_mix(n), reference(n))
+
+    def test_release_order(self):
+        # (Few enough iterations that the loop stays off tier 2.)
+        LOG.clear()
+        release_order(30)
+        expect = []
+        for i in range(30):
+            expect += [("bound", i), ("del", i), ("after", i)]
+        self.assertEqual(LOG, expect)
+
+    def test_raise_from_helper(self):
+        objs = [WithProp(1)] * 300 + [Plain()]
+        try:
+            raise_in_helper(objs)
+        except AttributeError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("no AttributeError")
+        while tb.tb_next is not None:
+            tb = tb.tb_next
+        self.assertEqual(tb.tb_frame.f_code.co_name, "raise_in_helper")
+        line = raise_in_helper.__code__.co_firstlineno + 3
+        self.assertEqual(tb.tb_lineno, line)
+
+    def test_line_events(self):
+        lines = []
+
+        def tracer(frame, event, arg):
+            if frame.f_code is handoff_mix.__code__ and event == "line":
+                lines.append(frame.f_lineno - handoff_mix.__code__.co_firstlineno)
+            return tracer
+
+        handoff_mix(50)
+        sys.settrace(tracer)
+        try:
+            r = handoff_mix(3)
+        finally:
+            sys.settrace(None)
+        self.assertEqual(r, handoff_mix(3))
+        # Every body line of each iteration, in order.
+        self.assertEqual(lines.count(10), 2)
+        self.assertEqual(lines[:4], [3, 4, 5, 6])
+
+
+if __name__ == "__main__":
+    unittest.main()

From fa91666bc947ccccea8cd1ba8e756ec4320b713a Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 00:39:45 -0700
Subject: [PATCH 127/173] perf: read a LOAD_GLOBAL site's value in place and
 fuse LOAD_FAST by table

- A LOAD_GLOBAL site's stamp slot also keeps the address of the value
  the core loop last read under the stamp (bit 0 marks a builtin). Dict
  stamps are unique to a dict and its state, and every mutable access
  draws a new one, so while the slot's stamp matches the frame's
  namespaces the value is still there: the arm reads it without decoding
  the inline cache or indexing the dict. Every other write of the stamp
  clears the address.
- The fused-pair table also records the receiver-fused shapes after a
  LOAD_FAST (a method call, an attribute read or chain, an attribute
  store), so a plain load of a heap local no longer decodes the next
  instruction to rule them out.
- The in-loop handoff derives the stack's base and capacity itself, and
  reloads when the run allocated a site table the loop holds an empty
  view of, so the call sites carry less code.

Adds tests/regrtest/test_core_global_cache.py (rebinding, deleting,
shadowing and replacing globals and builtins; one code object under
several globals dicts).

Measured (quick build, instructions retired, PYTHONHASHSEED=0):
- suite geomean 0.9953x of the previous commit (tomllib_loads 0.986x,
  chaos 0.990x, deepcopy 0.991x, logging_mix 0.991x, sudoku 0.991x,
  class_creation 0.992x).
- frame-JIT-off probes, per statement: x = G 146 -> 124,
  x = len(l) 203 -> 172, x = isinstance(a, int) 725 -> 663,
  x = math.pi 317 -> 295, x = l[1] 297 -> 278.
---
 crates/weavepy-vm/src/lib.rs             | 457 +++++++++++++----------
 tests/regrtest/test_core_global_cache.py | 118 ++++++
 2 files changed, 375 insertions(+), 200 deletions(-)
 create mode 100644 tests/regrtest/test_core_global_cache.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index cc49ffa8..ff103b4d 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -11018,33 +11018,41 @@ impl Interpreter {
     }
 
     /// [`Self::core_offload_run`] from inside [`Self::leaf_core`]'s loop, which
-    /// holds the running activation at `at` with its operand stack at
-    /// `base` (capacity `cap`): synced, then run. `true` when the loop goes
-    /// on in place, at the new `at`: the same activation, its stack where
-    /// it was, nothing pending (what a reload would derive again is
-    /// unchanged). Otherwise the loop reloads, which ends the burst first
-    /// if it must (`sw.pending`).
+    /// holds the running activation at `at`: synced, then run. `true` when
+    /// the loop goes on in place, at the new `at`: the same activation, its
+    /// stack where it was, its site tables as they were, nothing pending:
+    /// what a reload would derive again is unchanged. Otherwise the loop
+    /// reloads, which ends the burst first if it must (`sw.pending`).
     ///
     /// # Safety
     ///
-    /// `at`, `base` and `cap` are the running activation's, as the core
-    /// loop holds them (every stack slot below `at.len` initialized).
+    /// `at` is the running activation's, as the core loop holds it (every
+    /// stack slot below `at.len` initialized).
     #[inline(never)]
     unsafe fn core_offload_in_loop<const MODE: u8>(
         &mut self,
         sw: &mut CoreSwitch,
         snap_gen: u64,
-        base: *mut Object,
-        cap: usize,
         at: &mut CoreAt,
     ) -> bool {
         let (cur, last) = (sw.cur, sw.last);
-        // SAFETY: see `CoreSwitch`; the caller's contract.
-        unsafe {
-            (*cur).stack.set_len(at.len);
+        // SAFETY: see `CoreSwitch`; the caller's contract. (The core loop
+        // writes its operand stack in place, so its base and capacity are
+        // the vector's.)
+        let (base, cap) = unsafe {
+            let stack = &mut (*cur).stack;
+            stack.set_len(at.len);
             (*cur).pc = at.pc as u32;
             *last = at.last;
-        }
+            (stack.as_mut_ptr(), stack.capacity())
+        };
+        // The site tables the loop holds views of (empty until allocated).
+        let tables = |frame: &Frame| {
+            code_vm_ext(&frame.code)
+                .map(|e| (e.stamp_slots.is_allocated(), e.method_slots.is_allocated()))
+        };
+        // SAFETY: as above.
+        let tables_before = tables(unsafe { &*cur });
         if let Some(stop) = self.core_offload_run(sw, snap_gen, MODE) {
             // (What the burst's own end would drop with it.)
             sw.pending = Some(CoreExit::Stop(stop));
@@ -11056,12 +11064,13 @@ impl Interpreter {
         // SAFETY: the same activation, unborrowed here.
         let frame = unsafe { &mut *cur };
         let pc = frame.pc as usize;
-        // (A reload would also count an entry at pc 0 as a call, and check
-        // the pc against the code.)
+        // (A reload would also count an entry at pc 0 as a call, check the
+        // pc against the code, and view a site table the run allocated.)
         if frame.stack.as_mut_ptr() != base
             || frame.stack.capacity() != cap
             || pc == 0
             || pc >= frame.code.instructions.len()
+            || tables(frame) != tables_before
         {
             return false;
         }
@@ -11564,15 +11573,10 @@ impl Interpreter {
                 ($mode:expr) => {{
                     let mut at = CoreAt { len, pc, last };
                     // SAFETY: the running activation's state, as held here.
-                    if unsafe {
-                        self.core_offload_in_loop::<{ $mode }>(sw, snap_gen, base, cap, &mut at)
-                    } {
+                    if unsafe { self.core_offload_in_loop::<{ $mode }>(sw, snap_gen, &mut at) } {
                         len = at.len;
                         pc = at.pc;
                         last = at.last;
-                        // (The run may have allocated a site table.)
-                        cold_stamps = None;
-                        cold_mslots = None;
                         continue;
                     } else {
                         break Some(CoreExit::Reload);
@@ -11666,7 +11670,8 @@ impl Interpreter {
                         // every pc keeps its own unit, so the pair is fused
                         // here instead — one dispatch, and `x = y` never
                         // touches the stack at all.
-                        match fast_pairs.get(pc).copied().unwrap_or(0) {
+                        let kind = fast_pairs.get(pc).copied().unwrap_or(0);
+                        match kind {
                             1 => {
                                 // SAFETY: a pair kind is only recorded where
                                 // `pc + 1` names an instruction.
@@ -11743,7 +11748,7 @@ impl Interpreter {
                                 // pair kind is only recorded where the branch
                                 // instruction exists.
                                 let v = unsafe { &*lbase.add(i) };
-                                let jump_pc = pc + 1 + usize::from(fast_pairs[pc] == 4);
+                                let jump_pc = pc + 1 + usize::from(kind == 4);
                                 let jump = unsafe { *instrs.add(jump_pc) };
                                 let taken = match (jump.op, v) {
                                     (_, Object::Unbound) => None,
@@ -11793,111 +11798,104 @@ impl Interpreter {
                                 | Object::Bool(_)
                                 | Object::None) => scalar_copy(s),
                                 Object::Unbound => break None,
-                                other => {
-                                    // `x.attr` on a local receiver: read straight
-                                    // off the local (no receiver clone pushed and
-                                    // released); a miss loads the receiver and
-                                    // leaves `LOAD_ATTR` to its own arm.
-                                    // `x.m(...)` of a leaf or native method on the
-                                    // borrowed local (see `core_local_method`).
-                                    if pc + 1 < ninstrs
-                                        && len < cap
-                                        && (*instrs.add(pc + 1)).op == OpCode::LoadMethodAttr
-                                    {
-                                        if let Some((r, call_pc)) = self.core_local_method(
-                                            code,
-                                            ext,
-                                            other,
-                                            pc,
-                                            lbase,
-                                            nlocals,
-                                            consts,
-                                            sw.depth_cell,
-                                        ) {
-                                            last = call_pc;
-                                            pc = call_pc + 1;
-                                            match r {
-                                                Ok(v) => {
-                                                    base.add(len).write(v);
-                                                    len += 1;
-                                                    continue;
-                                                }
-                                                Err(e) => {
-                                                    break Some(CoreExit::Stop(LeafStop::Raised(
-                                                        e,
-                                                    )));
-                                                }
+                                // A plain load (kind 0) of a heap value is a
+                                // clone; the kinds past the pairs fuse the
+                                // instruction after it (see `code_fast_pairs`).
+                                other if kind < FAST_METHOD => clone_hot(other),
+                                // `x.m(...)` of a leaf or native method on the
+                                // borrowed local (see `core_local_method`).
+                                other if kind == FAST_METHOD => {
+                                    if let Some((r, call_pc)) = self.core_local_method(
+                                        code,
+                                        ext,
+                                        other,
+                                        pc,
+                                        lbase,
+                                        nlocals,
+                                        consts,
+                                        sw.depth_cell,
+                                    ) {
+                                        last = call_pc;
+                                        pc = call_pc + 1;
+                                        match r {
+                                            Ok(v) => {
+                                                base.add(len).write(v);
+                                                len += 1;
+                                                continue;
+                                            }
+                                            Err(e) => {
+                                                break Some(CoreExit::Stop(LeafStop::Raised(e)));
                                             }
                                         }
                                     }
-                                    if matches!(other, Object::Instance(_) | Object::Module(_))
-                                        && pc + 1 < ninstrs
-                                    {
-                                        let next = *instrs.add(pc + 1);
-                                        if next.op == OpCode::LoadAttr {
-                                            if code
-                                                .instructions
-                                                .get(pc + 2)
-                                                .is_some_and(|ins| ins.op == OpCode::LoadAttr)
-                                            {
-                                                if let Some((v, end)) =
-                                                    Self::core_local_attr_chain(code, other, pc + 1)
-                                                {
-                                                    base.add(len).write(v);
-                                                    len += 1;
-                                                    last = end;
-                                                    pc = end + 1;
-                                                    continue;
-                                                }
-                                            } else if let (Some(ext), Object::Instance(inst)) =
-                                                (ext, other)
-                                            {
-                                                // The site's field shortcut, in line (the
-                                                // common monomorphic read).
-                                                if let Some(v) = field_slot_hit(ext, pc + 1, inst) {
-                                                    base.add(len).write(clone_hot(v));
-                                                    len += 1;
-                                                    last = pc + 1;
-                                                    pc += 2;
-                                                    continue;
-                                                }
-                                            }
-                                            if let Some(v) = Self::core_local_attr(
-                                                ext,
-                                                code,
-                                                other,
-                                                pc + 1,
-                                                next.arg,
-                                            ) {
-                                                base.add(len).write(v);
+                                    clone_hot(other)
+                                }
+                                // `x.attr` on a local receiver: read straight off
+                                // the local (no receiver clone pushed and
+                                // released); a miss loads the receiver and leaves
+                                // `LOAD_ATTR` to its own arm.
+                                other @ (Object::Instance(_) | Object::Module(_)) => {
+                                    // SAFETY: an attribute kind is only recorded
+                                    // where `pc + 1` names an instruction.
+                                    let next = *instrs.add(pc + 1);
+                                    if kind == FAST_ATTR_CHAIN {
+                                        if let Some((v, end)) =
+                                            Self::core_local_attr_chain(code, other, pc + 1)
+                                        {
+                                            base.add(len).write(v);
+                                            len += 1;
+                                            last = end;
+                                            pc = end + 1;
+                                            continue;
+                                        }
+                                    } else if kind == FAST_ATTR {
+                                        // The site's field shortcut, in line (the
+                                        // common monomorphic read).
+                                        if let (Some(ext), Object::Instance(inst)) = (ext, other) {
+                                            if let Some(v) = field_slot_hit(ext, pc + 1, inst) {
+                                                base.add(len).write(clone_hot(v));
                                                 len += 1;
                                                 last = pc + 1;
                                                 pc += 2;
                                                 continue;
                                             }
                                         }
-                                        // `x.attr = v` on a local receiver:
-                                        // store straight into it, so the
-                                        // receiver is never pushed, cloned and
-                                        // dropped (a drop the collector grades).
-                                        if next.op == OpCode::StoreAttr
-                                            && len > 0
-                                            && Self::core_store_local_attr(
-                                                code,
-                                                other,
-                                                pc + 1,
-                                                next.arg,
-                                                &*base.add(len - 1),
-                                            )
-                                        {
-                                            len -= 1;
+                                    }
+                                    if kind != FAST_STORE_ATTR {
+                                        if let Some(v) = Self::core_local_attr(
+                                            ext,
+                                            code,
+                                            other,
+                                            pc + 1,
+                                            next.arg,
+                                        ) {
+                                            base.add(len).write(v);
+                                            len += 1;
                                             last = pc + 1;
                                             pc += 2;
                                             continue;
                                         }
+                                    } else if len > 0
+                                        // `x.attr = v` on a local receiver: store
+                                        // straight into it, so the receiver is
+                                        // never pushed, cloned and dropped (a
+                                        // drop the collector grades).
+                                        && Self::core_store_local_attr(
+                                            code,
+                                            other,
+                                            pc + 1,
+                                            next.arg,
+                                            &*base.add(len - 1),
+                                        )
+                                    {
+                                        len -= 1;
+                                        last = pc + 1;
+                                        pc += 2;
+                                        continue;
                                     }
                                     clone_hot(other)
                                 }
+                                other => clone_hot(other),
                             };
                             base.add(len).write(c);
                         }
@@ -14190,86 +14188,99 @@ impl Interpreter {
                         let (gdict, bdict) = (frame.globals.as_ptr(), frame.builtins.as_ptr());
                         let gid = specialize::rc_id(&frame.globals);
                         let g_stamp = unsafe { (*gdict).mutation_stamp() };
-                        let hit = match code.caches.get(pc as u32) {
-                            IC::LoadGlobalModule {
-                                globals_id,
-                                key_idx,
-                            } if globals_id == gid && slot.get() == [gid, g_stamp, 0] => unsafe {
-                                (*gdict).get_index(key_idx as usize)
-                            },
-                            IC::LoadGlobalBuiltin {
-                                builtins_id,
-                                key_idx,
-                            } if builtins_id == specialize::rc_id(&frame.builtins)
-                                && !self.globals_missing_any.get()
-                                && slot.get()
-                                    == [gid, g_stamp, unsafe { (*bdict).mutation_stamp() }] =>
-                            {
-                                let hit = unsafe { (*bdict).get_index(key_idx as usize) };
-                                // `len(local)`: the length straight off the local.
-                                if let Some((_, Object::Builtin(f))) = hit {
-                                    if Rc::as_ptr(f) as usize == self.leaf_fns().len_ptr
-                                        && pc + 3 < ninstrs
+                        let stamp = slot.get();
+                        // The value the site last read, in place while the
+                        // stamp holds (see `StampSlot::value`); else the
+                        // inline cache's entry under the stamp, recorded.
+                        let held = slot.value();
+                        let (v, builtin): (&Object, bool) = if held != 0
+                            && stamp[0] == gid
+                            && stamp[1] == g_stamp
+                            && if held & 1 == 0 {
+                                stamp[2] == 0
+                            } else {
+                                !self.globals_missing_any.get()
+                                    && stamp[2] == unsafe { (*bdict).mutation_stamp() }
+                            } {
+                            // SAFETY: the dicts are as they were when the
+                            // value was read (see `StampSlot::value`).
+                            (unsafe { &*((held & !1) as *const Object) }, held & 1 != 0)
+                        } else {
+                            let (hit, builtin) = match code.caches.get(pc as u32) {
+                                IC::LoadGlobalModule {
+                                    globals_id,
+                                    key_idx,
+                                } if globals_id == gid && stamp == [gid, g_stamp, 0] => {
+                                    (unsafe { (*gdict).get_index(key_idx as usize) }, false)
+                                }
+                                IC::LoadGlobalBuiltin {
+                                    builtins_id,
+                                    key_idx,
+                                } if builtins_id == specialize::rc_id(&frame.builtins)
+                                    && !self.globals_missing_any.get()
+                                    && stamp
+                                        == [gid, g_stamp, unsafe { (*bdict).mutation_stamp() }] =>
+                                {
+                                    (unsafe { (*bdict).get_index(key_idx as usize) }, true)
+                                }
+                                _ => offload!(OFFLOAD_HELPER),
+                            };
+                            let Some((_, v)) = hit else {
+                                offload!(OFFLOAD_HELPER);
+                            };
+                            slot.set_value(v, builtin);
+                            (v, builtin)
+                        };
+                        // `len(local)`: the length straight off the local.
+                        if builtin {
+                            if let Object::Builtin(f) = v {
+                                if Rc::as_ptr(f) as usize == self.leaf_fns().len_ptr
+                                    && pc + 3 < ninstrs
+                                {
+                                    // SAFETY: `pc + 3 < ninstrs`.
+                                    let (null, load, call) = unsafe {
+                                        (
+                                            *instrs.add(pc + 1),
+                                            *instrs.add(pc + 2),
+                                            *instrs.add(pc + 3),
+                                        )
+                                    };
+                                    if null.op == OpCode::PushNull
+                                        && load.op == OpCode::LoadFast
+                                        && call.op == OpCode::Call
+                                        && call.arg == 1
+                                        && (load.arg as usize) < nlocals
                                     {
-                                        // SAFETY: `pc + 3 < ninstrs`.
-                                        let (null, load, call) = unsafe {
-                                            (
-                                                *instrs.add(pc + 1),
-                                                *instrs.add(pc + 2),
-                                                *instrs.add(pc + 3),
-                                            )
-                                        };
-                                        if null.op == OpCode::PushNull
-                                            && load.op == OpCode::LoadFast
-                                            && call.op == OpCode::Call
-                                            && call.arg == 1
-                                            && (load.arg as usize) < nlocals
-                                        {
-                                            // SAFETY: `load.arg < nlocals`.
-                                            let n = match unsafe { &*lbase.add(load.arg as usize) }
-                                            {
-                                                // SAFETY: a read between two
-                                                // instructions.
-                                                Object::List(l) => {
-                                                    unsafe { l.peek() }.map(Vec::len)
-                                                }
-                                                Object::Tuple(t) => Some(t.len()),
-                                                Object::Str(s) => {
-                                                    Some(crate::object::str_char_len(s))
-                                                }
-                                                // SAFETY: as above.
-                                                Object::Dict(d) => {
-                                                    unsafe { d.peek() }.map(|d| d.len())
-                                                }
-                                                // A native `__len__` (a deque's).
-                                                v @ Object::Instance(_) => {
-                                                    match self.leaf_instance_len(v) {
-                                                        Some(Object::Int(n)) => {
-                                                            usize::try_from(n).ok()
-                                                        }
-                                                        _ => None,
-                                                    }
+                                        // SAFETY: `load.arg < nlocals`.
+                                        let n = match unsafe { &*lbase.add(load.arg as usize) } {
+                                            // SAFETY: a read between two
+                                            // instructions.
+                                            Object::List(l) => unsafe { l.peek() }.map(Vec::len),
+                                            Object::Tuple(t) => Some(t.len()),
+                                            Object::Str(s) => Some(crate::object::str_char_len(s)),
+                                            // SAFETY: as above.
+                                            Object::Dict(d) => unsafe { d.peek() }.map(|d| d.len()),
+                                            // A native `__len__` (a deque's).
+                                            v @ Object::Instance(_) => {
+                                                match self.leaf_instance_len(v) {
+                                                    Some(Object::Int(n)) => usize::try_from(n).ok(),
+                                                    _ => None,
                                                 }
-                                                _ => None,
-                                            };
-                                            if let Some(n) = n.and_then(|n| i64::try_from(n).ok()) {
-                                                // SAFETY: `len < cap`.
-                                                unsafe { base.add(len).write(int_hot(n)) };
-                                                len += 1;
-                                                last = pc + 3;
-                                                pc += 4;
-                                                continue;
                                             }
+                                            _ => None,
+                                        };
+                                        if let Some(n) = n.and_then(|n| i64::try_from(n).ok()) {
+                                            // SAFETY: `len < cap`.
+                                            unsafe { base.add(len).write(int_hot(n)) };
+                                            len += 1;
+                                            last = pc + 3;
+                                            pc += 4;
+                                            continue;
                                         }
                                     }
                                 }
-                                hit
                             }
-                            _ => offload!(OFFLOAD_HELPER),
-                        };
-                        let Some((_, v)) = hit else {
-                            offload!(OFFLOAD_HELPER);
-                        };
+                        }
                         let v = match v {
                             s @ (Object::Int(_)
                             | Object::Float(_)
@@ -72754,10 +72765,11 @@ struct CodeConstObjects {
     leaf_tries: std::sync::atomic::AtomicU64,
     /// Per-instruction fused-pair kinds (see [`code_fast_pairs`]): `1`
     /// at a `LOAD_FAST` whose successor is another `LOAD_FAST`, `2` when
-    /// it is a `STORE_FAST`, `0` everywhere else. CPython's compiler
-    /// fuses exactly these pairs into one instruction and the decoder
-    /// splits them back apart; the core loop reads one byte to run the
-    /// pair as a single dispatch. Empty when the code has no pair.
+    /// it is a `STORE_FAST`, `3` and `4` for a branch on the local, the
+    /// receiver-fused kinds from [`FAST_METHOD`] on, `0` everywhere else.
+    /// CPython's compiler fuses the first pairs into one instruction and
+    /// the decoder splits them back apart; the core loop reads one byte to
+    /// run the pair as a single dispatch. Empty when the code has none.
     fast_pairs: std::sync::OnceLock<Box<[u8]>>,
     /// Whether the code is a generator body fast steps run (see
     /// `gen_fast`): 0 not yet scanned, 1 no, 2 yes.
@@ -73348,8 +73360,14 @@ fn code_attr_poly(code: &CodeObject, cache_pc: u32) -> Option<&AttrPoly> {
 /// and (for a builtin hit) the builtins dict's stamp. While all three
 /// match, the dicts are exactly as they were when the site last ran its
 /// full guards, so the cached index can be read straight away.
-#[repr(transparent)]
-struct StampSlot(std::cell::UnsafeCell<[u64; 3]>);
+///
+/// The second field is the address of the value the core loop's
+/// `LOAD_GLOBAL` arm last read under the stamp (with bit 0 set for a
+/// builtin), `0` when none: a dict's stamp moves with every mutation, so
+/// while the stamp holds the value stays where it is (see
+/// [`Self::value`]). Every other write of the stamp clears it.
+#[repr(C)]
+struct StampSlot(std::cell::UnsafeCell<[u64; 3]>, std::cell::Cell<u64>);
 
 // SAFETY: as `MethodSlot`: read and written only from the dispatch loop
 // with the GIL held.
@@ -73358,7 +73376,7 @@ unsafe impl Sync for StampSlot {}
 
 impl StampSlot {
     const fn empty() -> Self {
-        Self(std::cell::UnsafeCell::new([0; 3]))
+        Self(std::cell::UnsafeCell::new([0; 3]), std::cell::Cell::new(0))
     }
 
     #[inline]
@@ -73372,6 +73390,25 @@ impl StampSlot {
         // SAFETY: GIL-serialized; the exclusive reference lives only for
         // the assignment.
         unsafe { *self.0.get() = v };
+        self.1.set(0);
+    }
+
+    /// The value the `LOAD_GLOBAL` site last read under its stamp, as its
+    /// address with bit 0 set for a builtin (`0`: none). It is the value
+    /// while the stamp matches the frame's namespaces: the module dict's,
+    /// and for a builtin also the builtins dict's (stamps are unique to a
+    /// dict and its state).
+    #[inline(always)]
+    fn value(&self) -> u64 {
+        self.1.get()
+    }
+
+    /// Record the value a `LOAD_GLOBAL` read under the stamp just proved
+    /// (see [`Self::value`]).
+    #[inline]
+    fn set_value(&self, v: &Object, builtin: bool) {
+        self.1
+            .set(std::ptr::from_ref(v) as u64 | u64::from(builtin));
     }
 }
 
@@ -74738,16 +74775,15 @@ pub(crate) fn code_is_pure_leaf_pub(code: &CodeObject) -> bool {
 }
 
 /// The fused-pair kind of every instruction (see
-/// [`CodeConstObjects::fast_pairs`]), built once per code object.
-/// Empty for code with no pair at all — the common shape, whose
-/// `LOAD_FAST` arm then reads nothing.
+/// [`CodeConstObjects::fast_pairs`]), built once per code object: past
+/// the local-to-local pairs, the receiver-fused kinds ([`FAST_METHOD`] and
+/// on), which `LOAD_FAST` reads instead of decoding the next instruction.
+/// Empty for code with none at all, whose `LOAD_FAST` arm then reads
+/// nothing.
 fn code_fast_pairs<'a>(code: &CodeObject, ext: Option<&'a CodeConstObjects>) -> &'a [u8] {
     let Some(ext) = ext else { return &[] };
     ext.fast_pairs.get_or_init(|| {
-        // (`WEAVEPY_NO_PAIRS`, a bisection aid, leaves every table empty.)
-        if crate::hot_gates::env_flags::no_pairs() {
-            return Box::new([]);
-        }
+        let no_pairs = crate::hot_gates::env_flags::no_pairs();
         let ops = &code.instructions;
         let mut any = false;
         let mut kinds = vec![0u8; ops.len()];
@@ -74756,6 +74792,19 @@ fn code_fast_pairs<'a>(code: &CodeObject, ext: Option<&'a CodeConstObjects>) ->
                 continue;
             }
             let kind = match ops.get(pc + 1).map(|i| i.op) {
+                // The receiver-fused shapes: a method call, an attribute
+                // read (a chain of them) or store on the local.
+                Some(OpCode::LoadMethodAttr) => FAST_METHOD,
+                Some(OpCode::LoadAttr)
+                    if ops.get(pc + 2).map(|i| i.op) == Some(OpCode::LoadAttr) =>
+                {
+                    FAST_ATTR_CHAIN
+                }
+                Some(OpCode::LoadAttr) => FAST_ATTR,
+                Some(OpCode::StoreAttr) => FAST_STORE_ATTR,
+                // (`WEAVEPY_NO_PAIRS`, a bisection aid, leaves out the
+                // local-to-local pairs.)
+                _ if no_pairs => 0,
                 // A second load whose own successor reads an attribute
                 // off it keeps the receiver-fused arm instead.
                 Some(OpCode::LoadFast) => u8::from(!matches!(
@@ -74787,6 +74836,14 @@ fn code_fast_pairs<'a>(code: &CodeObject, ext: Option<&'a CodeConstObjects>) ->
     })
 }
 
+/// [`code_fast_pairs`]'s receiver-fused kinds: the `LOAD_FAST` is followed
+/// by a `LOAD_METHOD_ATTR`, a `LOAD_ATTR` (followed by another, for the
+/// chain), or a `STORE_ATTR`.
+const FAST_METHOD: u8 = 5;
+const FAST_ATTR: u8 = 6;
+const FAST_ATTR_CHAIN: u8 = 7;
+const FAST_STORE_ATTR: u8 = 8;
+
 /// A leaf's parameter count: its positional parameters, its keyword-only
 /// ones, then its `**kwargs` dictionary when it has one (bound by keyword
 /// calls only; a leaf has no `*args`).
diff --git a/tests/regrtest/test_core_global_cache.py b/tests/regrtest/test_core_global_cache.py
new file mode 100644
index 00000000..07a5c9fd
--- /dev/null
+++ b/tests/regrtest/test_core_global_cache.py
@@ -0,0 +1,118 @@
+"""A `LOAD_GLOBAL` site's cached value follows every namespace change.
+
+The core loop keeps the value a `LOAD_GLOBAL` site last read, in place,
+while the module and builtins dicts are exactly as they were. Rebinding,
+deleting, shadowing or unshadowing a name, changing a builtin, and running
+the same code against another globals dict must each be seen at once.
+"""
+
+import builtins
+import unittest
+
+G = 1
+
+
+def read_g(n):
+    total = 0
+    for _ in range(n):
+        total += G
+    return total
+
+
+def read_len(xs, n):
+    total = 0
+    for _ in range(n):
+        total += len(xs)
+    return total
+
+
+def read_abs(n):
+    total = 0
+    for i in range(n):
+        total += abs(-i)
+    return total
+
+
+def read_missing():
+    return MISSING_NAME  # noqa: F821
+
+
+class GlobalCacheTest(unittest.TestCase):
+    def test_rebind(self):
+        global G
+        try:
+            for v in (1, 2, "x", None, 3.5, 7):
+                G = v
+                if isinstance(v, str):
+                    self.assertEqual(read_g(0), 0)
+                    continue
+                if v is None:
+                    with self.assertRaises(TypeError):
+                        read_g(3)
+                    continue
+                self.assertEqual(read_g(200), v * 200)
+        finally:
+            G = 1
+
+    def test_delete_and_restore(self):
+        global G
+        self.assertEqual(read_g(100), 100)
+        del G
+        try:
+            with self.assertRaises(NameError):
+                read_g(1)
+        finally:
+            G = 4
+        self.assertEqual(read_g(100), 400)
+        G = 1
+
+    def test_builtin_shadowing(self):
+        xs = [1, 2, 3]
+        self.assertEqual(read_len(xs, 300), 900)
+        globals()["len"] = lambda s: 10
+        try:
+            self.assertEqual(read_len(xs, 300), 3000)
+        finally:
+            del globals()["len"]
+        self.assertEqual(read_len(xs, 300), 900)
+
+    def test_builtin_replaced(self):
+        self.assertEqual(read_abs(300), sum(range(300)))
+        saved = builtins.abs
+        builtins.abs = lambda v: 1
+        try:
+            self.assertEqual(read_abs(300), 300)
+        finally:
+            builtins.abs = saved
+        self.assertEqual(read_abs(300), sum(range(300)))
+
+    def test_other_globals(self):
+        src = "def f(n):\n    t = 0\n    for _ in range(n):\n        t += K\n    return t\n"
+        code = compile(src, "<g>", "exec")
+        fs = []
+        for k in range(4):
+            ns = {"K": k + 1}
+            exec(code, ns)
+            fs.append((ns, ns["f"]))
+        for _ in range(3):
+            for k, (ns, f) in enumerate(fs):
+                self.assertEqual(f(150), (k + 1) * 150)
+        fs[2][0]["K"] = 100
+        self.assertEqual(fs[2][1](10), 1000)
+        self.assertEqual(fs[1][1](10), 20)
+
+    def test_missing_then_defined(self):
+        for _ in range(50):
+            with self.assertRaises(NameError):
+                read_missing()
+        globals()["MISSING_NAME"] = 5
+        try:
+            self.assertEqual(read_missing(), 5)
+        finally:
+            del globals()["MISSING_NAME"]
+        with self.assertRaises(NameError):
+            read_missing()
+
+
+if __name__ == "__main__":
+    unittest.main()

From 2b55177f5b47db4aa77683ddbc99e35978bbe07c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 01:28:58 -0700
Subject: [PATCH 128/173] perf: branch on a numeric compare without pushing its
 bool

COMPARE_OP of two ints, two floats or an exactly representable mix,
followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE, takes the branch in
the compare's arm: one dispatch, and the bool is never pushed and
popped. The pair's last-executed pc is the jump's, as when the two run
apart.

LOAD_GLOBAL writes a scalar value to the stack in its own arm: as one
of the value match's results, LLVM split the copy into field moves
through the stack.

Extends tests/regrtest/test_core_loop_handoffs.py (every comparison
kind, mixed and NaN operands, a traceback line after a fused branch).

Measured (quick build, instructions retired, PYTHONHASHSEED=0):
- frame-JIT-off probes, per statement: `if a < b: x = 1` 248 -> 218,
  x = G 123 -> 111 (both runs under the same load).
- suite geomean 1.0008x of the previous commit, within run-to-run noise
  (the previous commit's binary measured 1.0017x of itself an hour
  apart).
---
 crates/weavepy-vm/src/lib.rs              | 31 +++++++++++-
 tests/regrtest/test_core_loop_handoffs.py | 57 +++++++++++++++++++++++
 2 files changed, 87 insertions(+), 1 deletion(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ff103b4d..6b845941 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -12732,6 +12732,26 @@ impl Interpreter {
                             CompareKind::Gt => ord.is_gt(),
                             CompareKind::GtE => ord.is_ge(),
                         };
+                        // A branch on the result (`if a < b:`) runs with the
+                        // compare: the bool is never pushed. (A compare can
+                        // fall through, so it never ends the code.)
+                        // SAFETY: `pc + 1 < ninstrs` (see `code_dispatch_len`).
+                        let next = unsafe { *instrs.add(pc + 1) };
+                        if matches!(next.op, OpCode::PopJumpIfFalse | OpCode::PopJumpIfTrue) {
+                            // SAFETY: both operands are scalars (no drop owed).
+                            len -= 2;
+                            last = pc + 1;
+                            pc += 2;
+                            if r == (next.op == OpCode::PopJumpIfTrue) {
+                                pc += next.arg as usize;
+                            } else if
+                            // SAFETY: a branch never ends the code (see
+                            // `code_dispatch_len`).
+                            unsafe { (*instrs.add(pc)).op } == OpCode::NotTaken {
+                                pc += 1;
+                            }
+                            continue;
+                        }
                         // SAFETY: both operands are scalars (no drop owed);
                         // the result takes the lower one's slot.
                         len -= 1;
@@ -14282,10 +14302,19 @@ impl Interpreter {
                             }
                         }
                         let v = match v {
+                            // (Written here: a scalar copy that joins the
+                            // other arms' values is split into field moves.)
                             s @ (Object::Int(_)
                             | Object::Float(_)
                             | Object::Bool(_)
-                            | Object::None) => scalar_copy(s),
+                            | Object::None) => {
+                                // SAFETY: `len < cap`.
+                                unsafe { base.add(len).write(scalar_copy(s)) };
+                                len += 1;
+                                last = pc;
+                                pc += 1;
+                                continue;
+                            }
                             // `f(...)` of a pure leaf global function with
                             // simple arguments (see `core_pure_global_call`).
                             Object::Function(f)
diff --git a/tests/regrtest/test_core_loop_handoffs.py b/tests/regrtest/test_core_loop_handoffs.py
index aad069c3..d611b5c6 100644
--- a/tests/regrtest/test_core_loop_handoffs.py
+++ b/tests/regrtest/test_core_loop_handoffs.py
@@ -122,6 +122,63 @@ def test_invert_bool_warns(self):
         self.assertTrue(any(issubclass(x.category, DeprecationWarning) for x in w))
 
 
+def branch_counts(pairs):
+    counts = [0] * 6
+    for a, b in pairs:
+        if a < b:
+            counts[0] += 1
+        if a <= b:
+            counts[1] += 1
+        if a == b:
+            counts[2] += 1
+        if not a != b:
+            counts[3] += 1
+        if a > b:
+            counts[4] += 1
+        while a >= b:
+            counts[5] += 1
+            break
+    return counts
+
+
+def raise_after_compare(xs):
+    for x in xs:
+        if x > 2:
+            raise ValueError(x)  # line B
+    return None
+
+
+class CompareBranchTest(unittest.TestCase):
+    def test_counts(self):
+        nan = float("nan")
+        pairs = [(1, 2), (2, 1), (3, 3), (1.5, 2), (2, 1.5), (2.0, 2), (-1, -1.0),
+                 (2**53, float(2**53)), (2**53 + 1, float(2**53)), (nan, 1.0),
+                 (1.0, nan), (nan, nan), (True, 1), (False, 0.5), (-(2**63), 2**63 - 1)]
+        expect = [0] * 6
+        for a, b in pairs:
+            expect[0] += a < b
+            expect[1] += a <= b
+            expect[2] += a == b
+            expect[3] += not a != b
+            expect[4] += a > b
+            expect[5] += a >= b
+        for _ in range(100):
+            self.assertEqual(branch_counts(pairs), expect)
+
+    def test_traceback_line(self):
+        for _ in range(100):
+            self.assertIsNone(raise_after_compare([0, 1, 2]))
+        try:
+            raise_after_compare([0, 1, 2, 3])
+        except ValueError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("no ValueError")
+        while tb.tb_next is not None:
+            tb = tb.tb_next
+        self.assertEqual(tb.tb_lineno, raise_after_compare.__code__.co_firstlineno + 3)
+
+
 class HandoffTest(unittest.TestCase):
     def test_mixed_loop(self):
         def reference(n):

From 639efff30393754a2167e093a2f2b85430960aae Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 19:49:34 -0700
Subject: [PATCH 129/173] perf: keep a collapsed yield-from chain collapsed
 across yields

A resumed `yield from`/`await` chain already ran its innermost generator
directly, with the levels between parked in their `SEND`s, but every
yield put each level back in `Suspended` and the next resume walked and
re-validated the whole chain again: about 130 instructions per level per
item.

The levels now stay delegating when the innermost generator yields: the
chain's record (`ChainRec`) parks, owned by its levels, and the next
`SEND` of `None` to the outermost level resumes the innermost generator
directly, adding any generators it has since started delegating to.
`PyGenerator::state` is now the only way to reach a generator's state
outside the chain machinery, and it settles a parked chain first (each
level suspended in its `yield from`, as resuming level by level leaves
it), so everything that observes or changes a level (frames, locals,
`send`/`throw`/`close`, release and finalizers, the collector, tracing)
sees exactly the old state. A flush check by the pending list's position
replaces the per-level shell scan.

Measured (instructions retired per item, macOS):
- `for x in` a chain of plain `yield from` levels: depth 4 2150 -> 1878,
  depth 16 3746 -> 2048 (per level about 133 -> 14).
- An `await` chain driven by `send`: depth 16 7940 -> 6133.
- `sum()` over a 1024-node recursive tree generator: 5.97M -> 5.15M.
- Suite `generators_tree`: 2.01x -> 1.64x CPython; `coroutines`,
  `async_tree` and `comprehensions` unchanged.
---
 crates/weavepy-vm/src/frame_jit.rs            |   2 +-
 crates/weavepy-vm/src/gc_trace.rs             |   2 +-
 crates/weavepy-vm/src/gen_fast.rs             |   2 +-
 crates/weavepy-vm/src/lib.rs                  | 511 +++++++++++++-----
 crates/weavepy-vm/src/object.rs               |  46 +-
 crates/weavepy-vm/src/stdlib/gc_real.rs       |   2 +-
 crates/weavepy-vm/src/vm_singletons.rs        |   4 +-
 .../regrtest/test_yield_from_parked_chains.py | 412 ++++++++++++++
 8 files changed, 823 insertions(+), 158 deletions(-)
 create mode 100644 tests/regrtest/test_yield_from_parked_chains.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index acffbfff..6b322228 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -6115,7 +6115,7 @@ unsafe extern "C" fn h_await_iter(st: *mut State, at: *mut Object, kind: u64) ->
             match top {
                 // A read between instructions.
                 Object::Coroutine(g) => g
-                    .state
+                    .state()
                     .peek()
                     .is_some_and(|s| matches!(s, crate::object::GeneratorState::Created(_))),
                 // A `types.coroutine` generator is its own.
diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs
index ec58fd9e..1d4f8afc 100644
--- a/crates/weavepy-vm/src/gc_trace.rs
+++ b/crates/weavepy-vm/src/gc_trace.rs
@@ -3177,7 +3177,7 @@ pub fn clear_object_fields(obj: &Object) -> bool {
             // Dropping the suspended frame box breaks the cycle
             // (the finalizer — close() — has already run by the
             // time clear is reached; see collect phase 5c).
-            if let Ok(mut st) = g.state.try_borrow_mut() {
+            if let Ok(mut st) = g.state().try_borrow_mut() {
                 *st = crate::object::GeneratorState::Finished;
             }
         }
diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index db1f9cbc..093070f5 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -1025,7 +1025,7 @@ impl Interpreter {
         // runs while it's held.
         // SAFETY: nothing below reaches the cell again until `state`'s
         // last use (`peek_mut` rejects a live guard or shared cells).
-        let Some(state) = (unsafe { g.state.peek_mut() }) else {
+        let Some(state) = (unsafe { g.state().peek_mut() }) else {
             return GenNext::Declined;
         };
         let (GeneratorState::Suspended(boxed) | GeneratorState::Created(boxed)) = &*state else {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 6b845941..58b310d0 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -452,7 +452,7 @@ fn generator_frame_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) {
         Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => g,
         _ => return,
     };
-    let Ok(state) = g.state.try_borrow() else {
+    let Ok(state) = g.state_raw().try_borrow() else {
         return;
     };
     let boxed = match &*state {
@@ -1008,6 +1008,8 @@ pub struct Interpreter {
     /// ever dereferenced on the thread that pushed them, while the
     /// activations they name are live.
     lean_pending: Vec<usize>,
+    /// Spare [`ChainRec`]s (empty).
+    chain_pool: Vec<Box<ChainRec>>,
     /// The builtins the leaf burst may call directly (see [`LeafFns`]),
     /// resolved on first use.
     leaf_fns: std::cell::OnceCell<LeafFns>,
@@ -1260,6 +1262,7 @@ impl Default for Interpreter {
                 && !crate::hot_gates::env_flags::no_quiet()
                 && !Self::locals_fast_off(),
             lean_pending: Vec::new(),
+            chain_pool: Vec::new(),
             leaf_fns: std::cell::OnceCell::new(),
             leaf_opaque: ThreadCell::new((0, leaf_builtins::LeafMap::default())),
             core_next: ThreadCell::new(None),
@@ -1391,6 +1394,7 @@ impl Interpreter {
                 && !crate::hot_gates::env_flags::no_quiet()
                 && !Self::locals_fast_off(),
             lean_pending: Vec::new(),
+            chain_pool: Vec::new(),
             leaf_fns: std::cell::OnceCell::new(),
             leaf_opaque: ThreadCell::new((0, leaf_builtins::LeafMap::default())),
             core_next: ThreadCell::new(None),
@@ -3233,8 +3237,8 @@ impl Interpreter {
         // the raised warning routes through the unraisable hook,
         // with the coroutine itself as the hook's object.
         if let Object::Coroutine(g) = obj {
-            if matches!(&*g.state.borrow(), GeneratorState::Created(_)) {
-                *g.state.borrow_mut() = GeneratorState::Finished;
+            if matches!(&*g.state().borrow(), GeneratorState::Created(_)) {
+                *g.state().borrow_mut() = GeneratorState::Finished;
                 let qualname = g.qualname().to_str();
                 self.warn_unawaited_coroutine(obj, &qualname);
                 return true;
@@ -5861,7 +5865,7 @@ impl Interpreter {
         }
         let owner = py.gen_owner.borrow().as_ref().and_then(|w| w.upgrade());
         if let Some(g) = owner {
-            let state_kind = match &*g.state.borrow() {
+            let state_kind = match &*g.state().borrow() {
                 GeneratorState::Running | GeneratorState::Delegating(_) => 0u8,
                 GeneratorState::Suspended(_) => 1,
                 GeneratorState::Created(_) => 2,
@@ -5890,7 +5894,7 @@ impl Interpreter {
                     // Tear down the never-started generator: its frame
                     // locals (bound arguments) die now, like CPython's
                     // refcount-driven dealloc.
-                    *g.state.borrow_mut() = GeneratorState::Finished;
+                    *g.state().borrow_mut() = GeneratorState::Finished;
                     Self::wipe_frame_locals(py);
                     return Ok(Object::None);
                 }
@@ -6194,7 +6198,7 @@ impl Interpreter {
         if !matches!(g.code, Object::None) {
             return g.code.clone();
         }
-        match &*g.state.borrow() {
+        match &*g.state().borrow() {
             GeneratorState::Created(frame)
             | GeneratorState::Suspended(frame)
             | GeneratorState::Delegating(frame) => Object::Code(frame.code.clone()),
@@ -6209,7 +6213,7 @@ impl Interpreter {
     /// single `gi_frame` alive for the generator's lifetime). `None` once
     /// the generator has finished.
     fn gen_py_frame(&self, g: &Rc<PyGenerator>) -> Object {
-        let mut state = g.state.borrow_mut();
+        let mut state = g.state().borrow_mut();
         // A collapsed chain's generator is executing: its frame is on the
         // spine once anything flushed it (see `Self::gen_frame_attr`), and
         // otherwise as a suspended one's would be.
@@ -6304,7 +6308,7 @@ impl Interpreter {
     /// not delegating. A frame parked in a delegation sits just past the
     /// `SEND`/`YIELD_VALUE` pair with the delegate at top-of-stack.
     fn gen_yieldfrom(&self, g: &Rc<PyGenerator>) -> Object {
-        let state = g.state.borrow();
+        let state = g.state().borrow();
         if let GeneratorState::Suspended(frame) = &*state {
             {
                 let pc = frame.pc as usize;
@@ -7574,11 +7578,11 @@ impl Interpreter {
                     };
                     if done.gen.is_some() {
                         let result = self.inline_gen_finish(&mut done, exit);
-                        if !done.delegates.is_empty() {
+                        if done.chain.is_some() {
                             if matches!(result, Ok(GenStep::Yielded(_)))
-                                && !Self::collapse_flushed(&done)
+                                && !self.collapse_flushed(&done)
                             {
-                                self.core_uncollapse(&mut done);
+                                self.collapse_park(&mut done);
                             } else {
                                 // The chain's innermost level takes the
                                 // result (a yield on the spine, all of
@@ -7586,7 +7590,7 @@ impl Interpreter {
                                 let all = matches!(result, Ok(GenStep::Yielded(_)));
                                 let mut levels = Vec::new();
                                 let mut level = self.collapse_expand_one(&mut done);
-                                while all && !level.delegates.is_empty() {
+                                while all && level.chain.is_some() {
                                     let outer = self.collapse_expand_one(&mut level);
                                     levels.push(level);
                                     level = outer;
@@ -8176,7 +8180,7 @@ impl Interpreter {
     /// Park a finished inline slot (its callable already taken): release
     /// what the activation owned and return the slot to the pool.
     fn inline_park(&mut self, mut act: Box<InlineAct>) {
-        debug_assert!(!act.parked && act.delegates.is_empty());
+        debug_assert!(!act.parked && act.chain.is_none());
         let fr: &mut Frame = &mut act.frame;
         // Leftover operands drop before anything is reused (their drop
         // glue is arbitrary Rust code).
@@ -8245,7 +8249,7 @@ impl Interpreter {
     /// only the code handle and the slot's own fields remain.
     #[inline(always)]
     fn inline_park_clean(&mut self, mut act: Box<InlineAct>) {
-        debug_assert!(act.clean && act.frame.stack.is_empty() && act.delegates.is_empty());
+        debug_assert!(act.clean && act.frame.stack.is_empty() && act.chain.is_none());
         // SAFETY: as `inline_park`: the code handle is owned while active.
         unsafe { drop(std::ptr::read(&raw const act.frame.code)) };
         if act.owns_cells {
@@ -8400,7 +8404,7 @@ impl Interpreter {
         // runs while it is held; it ends before the activation resumes.
         // SAFETY: nothing below reaches the cell again until `state`'s last
         // use (`peek_mut` rejects a live guard or shared cells).
-        let state = unsafe { g.state.peek_mut() }?;
+        let state = unsafe { g.state().peek_mut() }?;
         let first_resume;
         {
             let boxed = match &*state {
@@ -8599,13 +8603,13 @@ impl Interpreter {
                 Ok(GenStep::Exhausted(v))
             }
             Ok(FrameOutcome::StartGenerator) => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 Err(RuntimeError::Internal(
                     "RETURN_GENERATOR in already-running generator".to_owned(),
                 ))
             }
             Err(err) => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 let escaped = self.pep479_escape(&gen, err);
                 let frame: &mut Frame = &mut boxed;
                 self.recycle_frame_allocs(frame);
@@ -14902,7 +14906,7 @@ impl Interpreter {
                         let fresh = len > 0
                             && match unsafe { &*base.add(len - 1) } {
                                 // SAFETY: a read between instructions.
-                                Object::Coroutine(g) => unsafe { g.state.peek() }
+                                Object::Coroutine(g) => unsafe { g.state().peek() }
                                     .is_some_and(|s| matches!(s, GeneratorState::Created(_))),
                                 // A `types.coroutine` generator is its own.
                                 Object::Generator(g) => {
@@ -20924,10 +20928,20 @@ impl Interpreter {
         let sent_none = matches!(frame.stack[n - 1], Object::None);
         // SAFETY: nothing below reaches the cell again until `state`'s last
         // use (`peek_mut` rejects a live guard or shared cells).
-        let Some(state) = (unsafe { g.state.peek_mut() }) else {
+        let Some(state) = (unsafe { g.state_raw().peek_mut() }) else {
             return SendHop::Declined;
         };
         let GeneratorState::Suspended(boxed) = &*state else {
+            // The outermost level of a parked chain: its innermost
+            // generator resumes directly (anything else settles it).
+            let chain = g.chain.get();
+            if chain != 0 && sent_none {
+                // SAFETY: a level's record is alive (see `ChainRec`).
+                let rec = unsafe { &*(chain as *const ChainRec) };
+                if rec.parked && std::ptr::eq(rec.levels[0].gen, Rc::as_ptr(g)) {
+                    return self.core_send_unpark(sw, send_pc, chain);
+                }
+            }
             return SendHop::Declined;
         };
         let gf: &Frame = boxed;
@@ -21031,7 +21045,7 @@ impl Interpreter {
             return None;
         }
         // SAFETY: a read between instructions; the view ends here.
-        let GeneratorState::Suspended(boxed) = (unsafe { g.state.peek_mut() })? else {
+        let GeneratorState::Suspended(boxed) = (unsafe { g.state_raw().peek_mut() })? else {
             return None;
         };
         let gf: &mut Frame = boxed;
@@ -21108,10 +21122,9 @@ impl Interpreter {
     /// parked in their own `SEND`s, marked executing
     /// ([`GeneratorState::Delegating`]), and wait on the pending list, so
     /// a flush puts them on the spine. A yield comes straight back to
-    /// this `SEND` ([`Self::core_uncollapse`] puts the levels back as
-    /// running them would have left them); a return or a raise resumes
-    /// only the innermost level ([`Self::collapse_expand_one`]). `false`
-    /// touches nothing.
+    /// this `SEND` and parks the chain ([`Self::collapse_park`]); a
+    /// return or a raise resumes only the innermost level
+    /// ([`Self::collapse_expand_one`]). `false` touches nothing.
     #[allow(clippy::too_many_arguments)]
     #[inline(never)]
     fn core_send_collapse(
@@ -21120,13 +21133,57 @@ impl Interpreter {
         send_pc: usize,
         first: *const PyGenerator,
         first_frame: *mut Frame,
-        mut inner: &Rc<PyGenerator>,
+        inner: &Rc<PyGenerator>,
+        level_send: usize,
+        inner_frame: *mut Frame,
+    ) -> bool {
+        let mut rec = self.chain_rec();
+        // SAFETY: `first` is suspended in a `yield from` of `inner`, both
+        // qualified by the caller.
+        let (inner, inner_frame) = unsafe {
+            Self::collapse_walk(
+                &mut rec,
+                first,
+                first_frame,
+                Rc::as_ptr(inner),
+                level_send,
+                inner_frame,
+            )
+        };
+        match self.collapse_commit(sw, send_pc, rec, inner, inner_frame) {
+            Ok(()) => true,
+            Err(mut rec) => {
+                for level in rec.levels.drain(..).rev() {
+                    // SAFETY: delegating since the walk.
+                    unsafe { level.suspend() };
+                }
+                self.chain_recycle(rec);
+                false
+            }
+        }
+    }
+
+    /// Add `g` (frame `gf`, suspended in the `yield from` at `level_send`
+    /// of `inner`, which `inner_frame` qualifies) to `rec` as its
+    /// innermost level, and every generator below it that qualifies in
+    /// turn, each made delegating just past its `SEND`
+    /// ([`Delegate::suspend`] undoes it). Returns the chain's innermost
+    /// generator, suspended, and its frame.
+    ///
+    /// # Safety
+    ///
+    /// As described: `g` and `inner` are alive (`inner` held by `g`'s
+    /// stack) and suspended, with no borrow of their state live.
+    #[inline(always)]
+    unsafe fn collapse_walk(
+        rec: &mut ChainRec,
+        mut g: *const PyGenerator,
+        mut gf: *mut Frame,
+        mut inner: *const PyGenerator,
         mut level_send: usize,
         mut inner_frame: *mut Frame,
-    ) -> bool {
-        let mut act = self.inline_slot();
-        debug_assert!(act.delegates.is_empty());
-        let (mut g, mut gf) = (first, first_frame);
+    ) -> (*const PyGenerator, *mut Frame) {
+        let addr = std::ptr::from_mut(rec) as usize;
         // The code objects whose lean resume a level already vouched for
         // (a recursive generator's levels share one).
         let mut code_ok: *const CodeObject = std::ptr::null();
@@ -21134,19 +21191,20 @@ impl Interpreter {
         // levels mostly share all three).
         let mut resend: (*const CodeObject, u32, usize) = (std::ptr::null(), u32::MAX, 0);
         loop {
-            // The level delegates from here (`Delegate::suspend` undoes it).
             // SAFETY: each level is alive (its holder is the level above,
             // or the running activation) and suspended (checked), with no
             // borrow of its state live; the box moves between variants
             // without drop glue.
             unsafe {
-                let st = (*g).state.as_ptr();
+                let st = (*g).state_raw().as_ptr();
                 let GeneratorState::Suspended(boxed) = std::ptr::read(st) else {
                     unreachable!("checked by `collapse_frame`");
                 };
                 std::ptr::write(st, GeneratorState::Delegating(boxed));
+                (*g).chain.set(addr);
+                (*gf).pc = level_send as u32 + 1;
             }
-            act.delegates.push(Delegate {
+            rec.levels.push(Delegate {
                 gen: g,
                 act: LeanAct {
                     frame: gf,
@@ -21154,7 +21212,7 @@ impl Interpreter {
                 },
                 send_pc: level_send,
             });
-            g = Rc::as_ptr(inner);
+            g = inner;
             gf = inner_frame;
             // SAFETY: as above.
             let Some((next, next_send)) = Self::collapse_next_memo(unsafe { &*gf }, &mut resend)
@@ -21164,12 +21222,28 @@ impl Interpreter {
             let Some(next_frame) = Self::collapse_frame_with(next, &mut code_ok) else {
                 break;
             };
-            inner = next;
+            inner = Rc::as_ptr(next);
             level_send = next_send;
             inner_frame = next_frame;
         }
+        (g, gf)
+    }
+
+    /// Run `inner` (frame `inner_frame`, suspended and qualified), the
+    /// innermost generator of the chain whose delegating levels `rec`
+    /// holds, as the delegate of the running activation's `SEND` of `None`
+    /// at `send_pc` (see [`Self::core_send_collapse`]). Past the recursion
+    /// limit, `rec` comes back untouched.
+    fn collapse_commit(
+        &mut self,
+        sw: &mut CoreSwitch,
+        send_pc: usize,
+        mut rec: Box<ChainRec>,
+        inner: *const PyGenerator,
+        inner_frame: *mut Frame,
+    ) -> Result<(), Box<ChainRec>> {
         // Each level counts against the recursion limit, as when resumed.
-        let levels = act.delegates.len();
+        let levels = rec.levels.len();
         let guard = if crate::recursion::enter_n(sw.depth_cell, levels) {
             match crate::recursion::enter_with(sw.depth_cell) {
                 crate::recursion::Enter::Ok(guard) => Some(guard),
@@ -21182,28 +21256,22 @@ impl Interpreter {
             None
         };
         let Some(guard) = guard else {
-            for level in act.delegates.drain(..) {
-                // SAFETY: delegating since above.
-                unsafe { level.suspend() };
-            }
-            if self.inline_pool.len() < INLINE_POOL_CAP {
-                self.inline_pool.push(act);
-            }
-            return false;
+            return Err(rec);
+        };
+        // Committed.
+        // SAFETY: alive (held by the innermost level's stack).
+        let inner = unsafe {
+            Rc::increment_strong_count(inner);
+            Rc::from_raw(inner)
         };
-        // Committed: each level is just past its `SEND`.
-        for level in &act.delegates {
-            // SAFETY: the level's frame, in its generator's box.
-            unsafe { (*level.act.frame).pc = level.send_pc as u32 + 1 };
-        }
-        let inner = inner.clone();
         // SAFETY: suspended (checked), no borrow of its state live.
         let GeneratorState::Suspended(mut boxed) = std::mem::replace(
-            unsafe { &mut *inner.state.as_ptr() },
+            unsafe { &mut *inner.state_raw().as_ptr() },
             GeneratorState::Running,
         ) else {
             unreachable!("checked by `collapse_frame`");
         };
+        debug_assert!(std::ptr::eq::<Frame>(&*boxed, inner_frame));
         boxed.gen_first_resume = false;
         // SAFETY: see `CoreSwitch`: the running activation is synced, its
         // stack `[.., first, None]` (checked by `core_send_hop`).
@@ -21214,7 +21282,9 @@ impl Interpreter {
         boxed.stack.push(Object::None);
         frame.pc = send_pc as u32 + 1;
         let gen_frame: *mut Frame = &raw mut *boxed;
+        let mut act = self.inline_slot();
         debug_assert!(act.gen.is_none() && act.gen_box.is_none() && act.guard.is_none());
+        debug_assert!(act.chain.is_none());
         // SAFETY: a pooled slot holds none of these (see `try_inline_gen`).
         unsafe {
             std::ptr::write(&raw mut act.gen, Some(inner));
@@ -21229,47 +21299,132 @@ impl Interpreter {
         act.exc_depth = self.exc_info_len();
         // The levels wait on the pending list, outermost first, as each
         // would while the next one ran (the buffer stays put from here).
-        for level in &mut act.delegates {
+        rec.pend_at = self.lean_pending.len();
+        self.lean_pending.reserve(levels);
+        for level in &mut rec.levels {
             self.lean_pending.push(&raw mut level.act as usize);
         }
+        act.chain = Some(rec);
         // SAFETY: see `CoreSwitch`.
         unsafe { push_fast(&mut *sw.inl, act) };
         sw.cur = gen_frame;
         sw.scratch = usize::MAX;
         sw.last = &raw mut sw.scratch;
-        true
+        Ok(())
     }
 
-    /// A collapsed chain's innermost generator yielded (`done`, its
-    /// finished activation, still holds the levels; the value goes to the
-    /// consumer): put the levels back as running them would have left
-    /// them, each suspended at its `yield from`. Only for levels without
-    /// a shell (see [`Self::collapse_expand_one`] otherwise).
-    #[inline]
-    fn core_uncollapse(&mut self, done: &mut InlineAct) {
-        let levels = done.delegates.len();
-        debug_assert!(done.delegates.iter().all(|l| l.act.shell.is_none()));
-        // Still pending means nothing flushed (a flush drains the list).
-        if let Some(innermost) = done.delegates.last_mut() {
-            let addr = &raw mut innermost.act as usize;
-            if self.lean_pending.last() == Some(&addr) {
-                let len = self.lean_pending.len();
-                self.lean_pending.truncate(len - levels);
+    /// The `SEND` of `None` at `send_pc` of `sw`'s (synced) running
+    /// activation to the outermost level of the parked chain whose record
+    /// is at `addr` (see [`ChainRec`]): its innermost generator resumes
+    /// directly, as [`Self::core_send_collapse`] resumes it, with any
+    /// generators that innermost one has since started delegating to
+    /// added as levels. Anything that doesn't qualify leaves the chain
+    /// parked (the `SEND`'s full handler settles it).
+    #[inline(never)]
+    fn core_send_unpark(&mut self, sw: &mut CoreSwitch, send_pc: usize, addr: usize) -> SendHop {
+        // SAFETY: a parked record owns itself (see `ChainRec`); taken
+        // here, put back below unless the chain runs.
+        let mut rec = unsafe { Box::from_raw(addr as *mut ChainRec) };
+        let repark = |rec: Box<ChainRec>| {
+            let _ = Box::into_raw(rec);
+            SendHop::Declined
+        };
+        let last = rec.levels.last().expect("a collapsed chain");
+        // SAFETY: the innermost level is alive and parked: its frame sits
+        // in its `SEND`, its delegate on top.
+        let (inner, inner_frame) = match unsafe { &*last.act.frame }.stack.last() {
+            Some(Object::Generator(g) | Object::Coroutine(g)) => match Self::collapse_frame(g) {
+                Some(frame) => (Rc::as_ptr(g), frame),
+                None => return repark(rec),
+            },
+            _ => return repark(rec),
+        };
+        // The innermost generator may since have started delegating
+        // (a `yield from` of a fresh generator): its own chain joins.
+        let base = rec.levels.len();
+        // SAFETY: as `core_send_collapse` (the innermost generator was
+        // just qualified, and with it the generator it delegates to).
+        let (inner, inner_frame) = match Self::collapse_next(unsafe { &*inner_frame }) {
+            Some((next, next_send)) => match Self::collapse_frame(next) {
+                Some(next_frame) => unsafe {
+                    Self::collapse_walk(
+                        &mut rec,
+                        inner,
+                        inner_frame,
+                        Rc::as_ptr(next),
+                        next_send,
+                        next_frame,
+                    )
+                },
+                None => (inner, inner_frame),
+            },
+            None => (inner, inner_frame),
+        };
+        rec.parked = false;
+        match self.collapse_commit(sw, send_pc, rec, inner, inner_frame) {
+            Ok(()) => SendHop::Collapsed,
+            Err(mut rec) => {
+                for level in rec.levels.drain(base..).rev() {
+                    // SAFETY: delegating since the walk.
+                    unsafe { level.suspend() };
+                }
+                rec.parked = true;
+                repark(rec)
             }
         }
-        for level in done.delegates.drain(..).rev() {
-            // SAFETY: the levels are alive and delegating (see
-            // `Delegate::gen`); nothing borrows their state here.
-            unsafe { level.suspend() };
+    }
+
+    /// A collapsed chain's innermost generator yielded (`done`, its
+    /// finished activation, holds the levels, none flushed; the value
+    /// goes to the consumer): the chain parks (see [`ChainRec`]), its
+    /// levels still delegating, off the pending list and the recursion
+    /// depth.
+    #[inline]
+    fn collapse_park(&mut self, done: &mut InlineAct) {
+        let mut rec = done.chain.take().expect("a collapsed chain");
+        debug_assert!(rec.levels.iter().all(|l| l.act.shell.is_none()));
+        debug_assert!(self.lean_pending.len() == rec.pend_at + rec.levels.len());
+        self.lean_pending.truncate(rec.pend_at);
+        crate::recursion::leave_n(crate::recursion::depth_cell(), rec.levels.len());
+        rec.parked = true;
+        let _ = Box::into_raw(rec);
+    }
+
+    /// An empty [`ChainRec`], from the pool or fresh.
+    #[inline]
+    fn chain_rec(&mut self) -> Box<ChainRec> {
+        self.chain_pool.pop().unwrap_or_else(|| {
+            Box::new(ChainRec {
+                levels: Vec::new(),
+                parked: false,
+                pend_at: 0,
+            })
+        })
+    }
+
+    /// Return an emptied [`ChainRec`] to the pool.
+    #[inline]
+    fn chain_recycle(&mut self, mut rec: Box<ChainRec>) {
+        debug_assert!(rec.levels.is_empty());
+        rec.parked = false;
+        if self.chain_pool.len() < 16 {
+            self.chain_pool.push(rec);
         }
-        crate::recursion::leave_n(crate::recursion::depth_cell(), levels);
     }
 
-    /// Whether any level of `act`'s collapsed chain has a shell (a flush
-    /// put the levels on the spine).
+    /// Whether a flush may have put the levels of `act`'s collapsed chain
+    /// on the spine (given them shells). Unflushed, the levels' entries
+    /// are still the pending list's newest ones where the chain put them
+    /// (a flush drains entries oldest first, and only the chain pushes
+    /// these).
     #[inline]
-    fn collapse_flushed(act: &InlineAct) -> bool {
-        act.delegates.iter().any(|l| l.act.shell.is_some())
+    fn collapse_flushed(&self, act: &InlineAct) -> bool {
+        let Some(rec) = &act.chain else {
+            return false;
+        };
+        let first = std::ptr::from_ref(&rec.levels[0].act) as usize;
+        self.lean_pending.len() != rec.pend_at + rec.levels.len()
+            || self.lean_pending.get(rec.pend_at) != Some(&first)
     }
 
     /// Resume the innermost level of `done`'s collapsed chain for real:
@@ -21279,12 +21434,15 @@ impl Interpreter {
     /// to), and the returned activation runs the level, taking over the
     /// consumer-facing fields and the rest of the chain.
     fn collapse_expand_one(&mut self, done: &mut InlineAct) -> Box<InlineAct> {
-        let addr = done
-            .delegates
+        let rec = done.chain.as_mut().expect("a collapsed chain");
+        let addr = rec
+            .levels
             .last_mut()
             .map(|l| &raw mut l.act as usize)
             .expect("a collapsed chain");
-        let level = done.delegates.pop().expect("a collapsed chain");
+        let level = rec.levels.pop().expect("a collapsed chain");
+        // SAFETY: alive, see `Delegate::gen`; it runs for real from here.
+        unsafe { (*level.gen).chain.set(0) };
         // The level's pending entry was the innermost generator's caller
         // entry (the delegate's `lean_pending_exit` takes it).
         let level_pending = (level.act.shell.is_none()).then_some(addr);
@@ -21297,12 +21455,12 @@ impl Interpreter {
         };
         // SAFETY: delegating (see `Delegate`); nothing borrows the state.
         let GeneratorState::Delegating(boxed) =
-            std::mem::replace(unsafe { &mut *gen.state.as_ptr() }, GeneratorState::Running)
+            std::mem::replace(unsafe { &mut *gen.state_raw().as_ptr() }, GeneratorState::Running)
         else {
             unreachable!("a collapsed level is delegating");
         };
         let mut act = self.inline_slot();
-        debug_assert!(act.delegates.is_empty());
+        debug_assert!(act.chain.is_none());
         let gen_frame = level.act.frame;
         // SAFETY: a pooled slot holds none of these (see `try_inline_gen`);
         // the level's recursion depth was entered by the collapse.
@@ -21323,7 +21481,11 @@ impl Interpreter {
         act.exhaust_arg = done.exhaust_arg;
         act.caller_pending = done.caller_pending.take();
         act.exc_depth = done.exc_depth;
-        std::mem::swap(&mut act.delegates, &mut done.delegates);
+        // The rest of the chain (if any) stays collapsed under the level.
+        match done.chain.take() {
+            Some(rec) if rec.levels.is_empty() => self.chain_recycle(rec),
+            rest => act.chain = rest,
+        }
         // SAFETY: the level's frame, just past its `SEND`.
         let jump = unsafe { &*gen_frame }.code.instructions[level.send_pc].arg;
         done.call_pc = level.send_pc;
@@ -21397,7 +21559,7 @@ impl Interpreter {
                 .shell
                 .as_deref()
                 .is_some_and(|s| !self.shell_pops_quietly(s, top.exc_depth))
-            || Self::collapse_flushed(top)
+            || self.collapse_flushed(top)
             || self.countdown_out(2)
             || crate::hot_gates::loop_gen() != snap_gen
         {
@@ -21432,10 +21594,10 @@ impl Interpreter {
             done.gen_frame = std::ptr::null_mut();
             Self::park_suspended_boxed(&gen, boxed);
             drop(gen);
-            // A collapsed chain's levels suspend again in their `yield
-            // from`s, the value passing them by.
-            if !done.delegates.is_empty() {
-                self.core_uncollapse(&mut done);
+            // A collapsed chain's levels stay collapsed, the value passing
+            // them by (see `Self::collapse_park`).
+            if done.chain.is_some() {
+                self.collapse_park(&mut done);
             }
             let call_pc = done.call_pc;
             let next_default = done.exhaust_arg == GEN_NEXT_DEFAULT;
@@ -21477,7 +21639,7 @@ impl Interpreter {
                             .shell
                             .as_deref()
                             .is_none_or(|s| self.shell_pops_quietly(s, t.exc_depth))
-                        && !Self::collapse_flushed(t)
+                        && !self.collapse_flushed(t)
                 })
                 && unsafe { &*cframe }
                     .code
@@ -21555,7 +21717,7 @@ impl Interpreter {
         drop(gen);
         // A collapsed chain's innermost level takes the return value: it
         // resumes in its `SEND`, the rest of the chain still collapsed.
-        if !done.delegates.is_empty() {
+        if done.chain.is_some() {
             let level = self.collapse_expand_one(&mut done);
             push_fast(inl, level);
         }
@@ -33545,7 +33707,7 @@ impl Interpreter {
                         // Currently executing (illegal to re-enter)?
                         "running" => {
                             return Ok(Object::Bool(matches!(
-                                &*g.state.borrow(),
+                                &*g.state().borrow(),
                                 GeneratorState::Running | GeneratorState::Delegating(_)
                             )))
                         }
@@ -33554,14 +33716,14 @@ impl Interpreter {
                         "frame" => {
                             // A collapsed chain's generator gets its spine
                             // entry first, as an executing frame has one.
-                            if matches!(&*g.state.borrow(), GeneratorState::Delegating(_)) {
+                            if matches!(&*g.state().borrow(), GeneratorState::Delegating(_)) {
                                 self.flush_pending_callers();
                             }
                             return Ok(self.gen_py_frame(g));
                         }
                         "suspended" => {
                             return Ok(Object::Bool(matches!(
-                                &*g.state.borrow(),
+                                &*g.state().borrow(),
                                 GeneratorState::Suspended(_)
                             )))
                         }
@@ -42032,7 +42194,7 @@ impl Interpreter {
                 // CPython GET_AWAITABLE: a coroutine that is suspended
                 // inside its own `await` (it has a yield-from
                 // sub-iterator) cannot gain a second awaiter.
-                let busy = matches!(&*g.state.borrow(), GeneratorState::Suspended(frame)
+                let busy = matches!(&*g.state().borrow(), GeneratorState::Suspended(frame)
                     if detect_yield_from_subiter(frame).is_some());
                 if busy {
                     return Err(crate::error::runtime_error(
@@ -43235,10 +43397,10 @@ impl Interpreter {
         // now (e.g. `await anext(me)` from inside the body) both count
         // as "already running" in CPython's `ag_running_async` sense.
         let mid_await = matches!(
-            &*g.state.borrow(),
+            &*g.state().borrow(),
             GeneratorState::Suspended(frame) if !frame.agen_yielded_value
         ) || matches!(
-            &*g.state.borrow(),
+            &*g.state().borrow(),
             GeneratorState::Running | GeneratorState::Delegating(_)
         );
         if !mid_await {
@@ -43364,7 +43526,7 @@ impl Interpreter {
     /// "completes with value" behavior) when the frame can't be inspected
     /// (e.g. the generator already finished).
     fn agen_yielded_a_value(g: &Rc<PyGenerator>) -> bool {
-        match &*g.state.borrow() {
+        match &*g.state().borrow() {
             GeneratorState::Suspended(frame) => frame.agen_yielded_value,
             _ => true,
         }
@@ -43539,11 +43701,11 @@ impl Interpreter {
         gen: &Rc<PyGenerator>,
         mut exc: PyException,
     ) -> Result<Object, RuntimeError> {
-        let prev_state = std::mem::replace(&mut *gen.state.borrow_mut(), GeneratorState::Running);
+        let prev_state = std::mem::replace(&mut *gen.state().borrow_mut(), GeneratorState::Running);
         let mut frame = match prev_state {
             GeneratorState::Created(boxed) | GeneratorState::Suspended(boxed) => *boxed,
             GeneratorState::Finished => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 // bpo-25887: throwing into an exhausted coroutine is a
                 // RuntimeError (matching `send`), except for the
                 // GeneratorExit that `close()` injects.
@@ -43563,7 +43725,7 @@ impl Interpreter {
                 )));
             }
             delegating @ GeneratorState::Delegating(_) => {
-                *gen.state.borrow_mut() = delegating;
+                *gen.state().borrow_mut() = delegating;
                 return Err(value_error(format!(
                     "{} already executing",
                     gen.kind.word()
@@ -43611,7 +43773,7 @@ impl Interpreter {
                         self.resume_outer_with_exc(gen, frame, close_exc)
                     }
                     Err(err) => {
-                        *gen.state.borrow_mut() = GeneratorState::Finished;
+                        *gen.state().borrow_mut() = GeneratorState::Finished;
                         self.recycle_frame_allocs(&mut frame);
                         Err(err)
                     }
@@ -43657,18 +43819,18 @@ impl Interpreter {
                             Ok(v)
                         }
                         Ok(FrameOutcome::Returned(v)) => {
-                            *gen.state.borrow_mut() = GeneratorState::Finished;
+                            *gen.state().borrow_mut() = GeneratorState::Finished;
                             self.recycle_frame_allocs(&mut frame);
                             Err(stop_iteration_with(v))
                         }
                         Ok(FrameOutcome::StartGenerator) => {
-                            *gen.state.borrow_mut() = GeneratorState::Finished;
+                            *gen.state().borrow_mut() = GeneratorState::Finished;
                             Err(RuntimeError::Internal(
                                 "RETURN_GENERATOR inside generator_throw".to_owned(),
                             ))
                         }
                         Err(err) => {
-                            *gen.state.borrow_mut() = GeneratorState::Finished;
+                            *gen.state().borrow_mut() = GeneratorState::Finished;
                             let escaped = self.pep479_escape(gen, err);
                             self.recycle_frame_allocs(&mut frame);
                             Err(escaped)
@@ -43686,7 +43848,7 @@ impl Interpreter {
                     return self.resume_outer_with_exc(gen, frame, inner_exc);
                 }
                 Err(err) => {
-                    *gen.state.borrow_mut() = GeneratorState::Finished;
+                    *gen.state().borrow_mut() = GeneratorState::Finished;
                     self.recycle_frame_allocs(&mut frame);
                     return Err(err);
                 }
@@ -43726,7 +43888,7 @@ impl Interpreter {
                 .and_then(|()| self.fire_exception_event(&py, &exc, true));
             self.pop_frame_shell();
             if let Err(err) = hook_result {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 return Err(err);
             }
             frame.suppress_call_event = true;
@@ -43753,7 +43915,7 @@ impl Interpreter {
                 // raise/handled pairing).
                 if let Some(inst) = handled_arg {
                     if let Err(err) = self.fire_gen_handled_event(&mut frame, inst) {
-                        *gen.state.borrow_mut() = GeneratorState::Finished;
+                        *gen.state().borrow_mut() = GeneratorState::Finished;
                         return Err(err);
                     }
                 }
@@ -43763,18 +43925,18 @@ impl Interpreter {
                         Ok(v)
                     }
                     Ok(FrameOutcome::Returned(v)) => {
-                        *gen.state.borrow_mut() = GeneratorState::Finished;
+                        *gen.state().borrow_mut() = GeneratorState::Finished;
                         self.recycle_frame_allocs(&mut frame);
                         Err(stop_iteration_with(v))
                     }
                     Ok(FrameOutcome::StartGenerator) => {
-                        *gen.state.borrow_mut() = GeneratorState::Finished;
+                        *gen.state().borrow_mut() = GeneratorState::Finished;
                         Err(RuntimeError::Internal(
                             "RETURN_GENERATOR inside generator_throw".to_owned(),
                         ))
                     }
                     Err(err) => {
-                        *gen.state.borrow_mut() = GeneratorState::Finished;
+                        *gen.state().borrow_mut() = GeneratorState::Finished;
                         let escaped = self.pep479_escape(gen, err);
                         self.recycle_frame_allocs(&mut frame);
                         Err(escaped)
@@ -43789,7 +43951,7 @@ impl Interpreter {
                 // `None` to both settrace and setprofile) must fire now —
                 // `gen_close` on a bare genexpr is exactly this path
                 // (test_cprofile.test_throw).
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 if crate::trace::any_observers_active() {
                     let py = frame.py_frame.clone().unwrap_or_else(|| {
                         let py = self.build_py_frame(&frame, None);
@@ -43848,7 +44010,7 @@ impl Interpreter {
     /// closes this way on the spot instead of queueing its finalizer.
     pub(crate) fn gen_close_quiet_detached(g: &PyGenerator) -> Option<Box<Frame>> {
         const RESUME_DEPTH1_MASK: u32 = 4;
-        let Ok(mut state) = g.state.try_borrow_mut() else {
+        let Ok(mut state) = g.state().try_borrow_mut() else {
             return None;
         };
         let (created, boxed) = match &mut *state {
@@ -43949,7 +44111,7 @@ impl Interpreter {
                 // a traceback pointing at the generator's frame (the
                 // unraisable hook asserts `exc_traceback is not None`).
                 let frame_obj = self.gen_py_frame(&g);
-                *g.state.borrow_mut() = GeneratorState::Finished;
+                *g.state().borrow_mut() = GeneratorState::Finished;
                 let err =
                     crate::error::runtime_error(format!("{} ignored GeneratorExit", g.kind.word()));
                 if let RuntimeError::PyException(pyexc) = &err {
@@ -43972,7 +44134,7 @@ impl Interpreter {
                     || exc.type_name() == "StopIteration"
                     || exc.type_name() == "StopAsyncIteration" =>
             {
-                *g.state.borrow_mut() = GeneratorState::Finished;
+                *g.state().borrow_mut() = GeneratorState::Finished;
                 // Python 3.13 (gh-104770): if the generator catches
                 // `GeneratorExit` and then *returns* a value, `close()`
                 // returns it. The return surfaces as `StopIteration(value)`;
@@ -43987,7 +44149,7 @@ impl Interpreter {
                 Ok(Object::None)
             }
             Err(err) => {
-                *g.state.borrow_mut() = GeneratorState::Finished;
+                *g.state().borrow_mut() = GeneratorState::Finished;
                 Err(err)
             }
         }
@@ -44118,7 +44280,7 @@ impl Interpreter {
                 // (CPython's exception_unwind → monitor_handled).
                 if let Some(inst) = handled_arg {
                     if let Err(err) = self.fire_gen_handled_event(&mut frame, inst) {
-                        *gen.state.borrow_mut() = GeneratorState::Finished;
+                        *gen.state().borrow_mut() = GeneratorState::Finished;
                         return Err(err);
                     }
                 }
@@ -44128,18 +44290,18 @@ impl Interpreter {
                         Ok(v)
                     }
                     Ok(FrameOutcome::Returned(v)) => {
-                        *gen.state.borrow_mut() = GeneratorState::Finished;
+                        *gen.state().borrow_mut() = GeneratorState::Finished;
                         self.recycle_frame_allocs(&mut frame);
                         Err(stop_iteration_with(v))
                     }
                     Ok(FrameOutcome::StartGenerator) => {
-                        *gen.state.borrow_mut() = GeneratorState::Finished;
+                        *gen.state().borrow_mut() = GeneratorState::Finished;
                         Err(RuntimeError::Internal(
                             "RETURN_GENERATOR inside resume_outer_with_exc".to_owned(),
                         ))
                     }
                     Err(err) => {
-                        *gen.state.borrow_mut() = GeneratorState::Finished;
+                        *gen.state().borrow_mut() = GeneratorState::Finished;
                         let escaped = self.pep479_escape(gen, err);
                         self.recycle_frame_allocs(&mut frame);
                         Err(escaped)
@@ -44148,7 +44310,7 @@ impl Interpreter {
             }
             Ok(None) => unreachable!(),
             Err(err) => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 let escaped = self.pep479_escape(gen, err);
                 self.recycle_frame_allocs(&mut frame);
                 Err(escaped)
@@ -44281,7 +44443,7 @@ impl Interpreter {
         // (only a `None` may be sent into it; the general path raises).
         // Validate and take the frame under one borrow. No Python runs
         // while it is held; release it before resuming the activation.
-        let mut state = gen.state.try_borrow_mut().ok()?;
+        let mut state = gen.state().try_borrow_mut().ok()?;
         let first_resume;
         {
             let boxed = match &*state {
@@ -44433,13 +44595,13 @@ impl Interpreter {
                 Self::generator_returned(v, raw_return)
             }
             Ok(FrameOutcome::StartGenerator) => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 Err(RuntimeError::Internal(
                     "RETURN_GENERATOR in already-running generator".to_owned(),
                 ))
             }
             Err(err) => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 let escaped = self.pep479_escape(gen, err);
                 let frame: &mut Frame = &mut boxed;
                 self.recycle_frame_allocs(frame);
@@ -44478,12 +44640,12 @@ impl Interpreter {
         // SAFETY: the store runs no code (the displaced state is `Running`,
         // which owns nothing) and no guard is live on the cell (`peek_mut`
         // checks).
-        match unsafe { gen.state.peek_mut() } {
+        match unsafe { gen.state().peek_mut() } {
             Some(state) if matches!(state, GeneratorState::Running) => unsafe {
                 std::ptr::write(state, GeneratorState::Finished);
             },
             Some(state) => *state = GeneratorState::Finished,
-            None => *gen.state.borrow_mut() = GeneratorState::Finished,
+            None => *gen.state().borrow_mut() = GeneratorState::Finished,
         }
     }
 
@@ -44509,14 +44671,14 @@ impl Interpreter {
         // SAFETY: the store runs no code (the displaced state is
         // `Running`, which owns nothing) and no guard is live on the cell
         // (`peek_mut` checks).
-        match unsafe { gen.state.peek_mut() } {
+        match unsafe { gen.state().peek_mut() } {
             // `Running` owns nothing: no drop glue for the displaced state.
             // SAFETY: as above.
             Some(state) if matches!(state, GeneratorState::Running) => unsafe {
                 std::ptr::write(state, GeneratorState::Suspended(boxed));
             },
             Some(state) => *state = GeneratorState::Suspended(boxed),
-            None => *gen.state.borrow_mut() = GeneratorState::Suspended(boxed),
+            None => *gen.state().borrow_mut() = GeneratorState::Suspended(boxed),
         }
     }
 
@@ -44612,12 +44774,12 @@ impl Interpreter {
         // Take the boxed frame; it is run *in place* (RFC 0069 WS4) so
         // the yield round-trip neither reallocates the box nor moves
         // the frame through the stack.
-        let prev_state = std::mem::replace(&mut *gen.state.borrow_mut(), GeneratorState::Running);
+        let prev_state = std::mem::replace(&mut *gen.state().borrow_mut(), GeneratorState::Running);
         let (mut boxed, first_resume) = match prev_state {
             GeneratorState::Created(boxed) => (boxed, true),
             GeneratorState::Suspended(boxed) => (boxed, false),
             GeneratorState::Finished => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 // bpo-25887: re-awaiting a completed coroutine is an
                 // error, not a silent StopIteration.
                 if matches!(gen.kind, crate::object::CoroutineKind::Coroutine) {
@@ -44634,7 +44796,7 @@ impl Interpreter {
                 )));
             }
             delegating @ GeneratorState::Delegating(_) => {
-                *gen.state.borrow_mut() = delegating;
+                *gen.state().borrow_mut() = delegating;
                 return Err(value_error(format!(
                     "{} already executing",
                     gen.kind.word()
@@ -44708,7 +44870,7 @@ impl Interpreter {
                 // dance unwraps to `None` rather than to the empty
                 // string we get from `from_builtin("StopIteration",
                 // "")`.
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 let frame: &mut Frame = &mut boxed;
                 // RFC 0059 WS4: the exhausted frame's storage is dead —
                 // donate it back to the frame pools.
@@ -44717,13 +44879,13 @@ impl Interpreter {
                 Self::generator_returned(v, raw_return)
             }
             Ok(FrameOutcome::StartGenerator) => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 Err(RuntimeError::Internal(
                     "RETURN_GENERATOR in already-running generator".to_owned(),
                 ))
             }
             Err(err) => {
-                *gen.state.borrow_mut() = GeneratorState::Finished;
+                *gen.state().borrow_mut() = GeneratorState::Finished;
                 let escaped = self.pep479_escape(gen, err);
                 let frame: &mut Frame = &mut boxed;
                 self.recycle_frame_allocs(frame);
@@ -57455,7 +57617,7 @@ impl Interpreter {
     /// inherits it so a *running* generator's `gi_frame` can be found
     /// without eager `PyFrame` construction.
     fn set_frame_gen_owner(gen: &Rc<PyGenerator>) {
-        if let GeneratorState::Created(boxed) = &mut *gen.state.borrow_mut() {
+        if let GeneratorState::Created(boxed) = &mut *gen.state().borrow_mut() {
             boxed.gen_owner = Some(Rc::downgrade(gen));
         }
     }
@@ -57909,7 +58071,7 @@ impl Interpreter {
         };
         let g = g.clone();
         // SAFETY: just made; nothing else reaches its state yet.
-        let state = unsafe { &mut *g.state.as_ptr() };
+        let state = unsafe { &mut *g.state().as_ptr() };
         if !matches!(state, GeneratorState::Created(boxed) if Self::lean_gen_code_ok(&boxed.code)) {
             return false;
         }
@@ -66225,11 +66387,61 @@ struct InlineAct {
     discard: bool,
     /// A collapsed `yield from` chain (see
     /// `Interpreter::core_send_collapse`): the generators between the
-    /// consumer and the one this activation runs, outermost first. The
-    /// activation's consumer-facing fields (`call_pc`, `exhaust_arg`,
-    /// `caller_pending`) are the consumer's. Empty otherwise; a pooled
-    /// slot keeps the buffer.
-    delegates: Vec<Delegate>,
+    /// consumer and the one this activation runs, outermost first (never
+    /// none). The activation's consumer-facing fields (`call_pc`,
+    /// `exhaust_arg`, `caller_pending`) are the consumer's.
+    chain: Option<Box<ChainRec>>,
+}
+
+/// The levels of a collapsed `yield from` chain (see
+/// `Interpreter::core_send_collapse`), outermost first, each in
+/// [`GeneratorState::Delegating`] with the record's address in its
+/// [`PyGenerator::chain`]. While the chain runs, the activation of its
+/// innermost generator owns the record ([`InlineAct::chain`]). When that
+/// generator yields, the chain parks (`Interpreter::collapse_park`): the
+/// record owns itself and the levels stay delegating, as if each had
+/// passed the value up and resumed straight back into its `SEND`. The
+/// next `SEND` of `None` to the outermost level resumes the innermost
+/// generator directly (`Interpreter::core_send_unpark`); anything else
+/// that reaches a level's state first settles the chain
+/// ([`settle_parked_chain`]), suspending each level in its `yield from`.
+struct ChainRec {
+    levels: Vec<Delegate>,
+    /// Parked (see above), or owned by a running activation.
+    parked: bool,
+    /// Where the levels' entries start on the pending list while the
+    /// chain runs unflushed (see `Interpreter::collapse_flushed`).
+    pend_at: usize,
+}
+
+// SAFETY: a record moves between threads with its generators, under the
+// GIL; a pooled one is empty.
+unsafe impl Send for ChainRec {}
+
+/// Settle the parked chain whose record is at `addr` (see [`ChainRec`]):
+/// each level suspends in its `yield from`, as resuming the chain level
+/// by level leaves it. A running chain's record is left alone (its
+/// levels are executing).
+#[cold]
+#[inline(never)]
+pub(crate) fn settle_parked_chain(addr: usize) {
+    // SAFETY: a level's `chain` names a live record: whoever ends the
+    // record clears it on every level first.
+    unsafe {
+        let rec = addr as *mut ChainRec;
+        if !(*rec).parked {
+            return;
+        }
+        let rec = Box::from_raw(rec);
+        // (Innermost first: a level's state is only read through the
+        // level above, which is still delegating meanwhile.)
+        for level in rec.levels.iter().rev() {
+            // The levels are alive (each held by the one above, the
+            // outermost by its owners) and parked: nothing borrows their
+            // state.
+            level.suspend();
+        }
+    }
 }
 
 /// One suspended level of a collapsed `yield from` chain (see
@@ -66249,19 +66461,22 @@ struct Delegate {
 }
 
 impl Delegate {
-    /// Put the level's generator back in `Suspended`, at its `RESUME`.
+    /// Put the level's generator back in `Suspended`, at its `RESUME`,
+    /// out of its chain.
     ///
     /// # Safety
     ///
     /// The generator is alive and in `Delegating`, with no borrow of its
     /// state live.
     unsafe fn suspend(&self) {
+        // SAFETY: alive (the caller's contract).
+        unsafe { (*self.gen).chain.set(0) };
         // SAFETY: the frame lives in the generator's box (see above).
         unsafe { (*self.act.frame).pc = self.send_pc as u32 + 2 };
         // SAFETY: the caller's contract; the box moves between variants
         // without drop glue.
         unsafe {
-            let st = (*self.gen).state.as_ptr();
+            let st = (*self.gen).state_raw().as_ptr();
             let GeneratorState::Delegating(boxed) = std::ptr::read(st) else {
                 unreachable!("a collapsed level is delegating");
             };
@@ -66315,7 +66530,7 @@ impl InlineAct {
             direct: false,
             binop: None,
             discard: false,
-            delegates: Vec::new(),
+            chain: None,
         });
         act.act.frame = std::ptr::from_mut::<Frame>(&mut act.frame);
         // SAFETY: parked slots hold a stale code copy (see the type docs):
@@ -66333,10 +66548,12 @@ unsafe impl Send for InlineAct {}
 impl Drop for InlineAct {
     fn drop(&mut self) {
         // An activation torn down mid-chain leaves its levels suspended.
-        for level in self.delegates.drain(..).rev() {
-            // SAFETY: the levels are alive (see `Delegate::gen`) and
-            // delegating; nothing borrows their state during a drop.
-            unsafe { level.suspend() };
+        if let Some(rec) = self.chain.take() {
+            for level in rec.levels.iter().rev() {
+                // SAFETY: the levels are alive (see `Delegate::gen`) and
+                // delegating; nothing borrows their state during a drop.
+                unsafe { level.suspend() };
+            }
         }
         if self.owns_cells {
             // SAFETY: owned cells are released once (the frame's own drop
@@ -75404,7 +75621,7 @@ pub(crate) fn code_gen_names(code: &CodeObject) -> (Object, Object) {
 /// outlives its young set, so a long-lived one (a parked coroutine, say)
 /// holds no more than it needs.
 pub(crate) fn compact_generator_frame(gen: &PyGenerator) {
-    let Ok(mut state) = gen.state.try_borrow_mut() else {
+    let Ok(mut state) = gen.state().try_borrow_mut() else {
         return;
     };
     let (GeneratorState::Created(frame) | GeneratorState::Suspended(frame)) = &mut *state else {
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 3a40b6c2..3b7a6a55 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -5504,7 +5504,8 @@ pub struct PyGenerator {
     /// strong reference) so it stays readable after the generator
     /// finishes and the frame is dropped.
     pub code: Object,
-    pub state: RefCell<GeneratorState>,
+    /// The execution state, read through [`Self::state`].
+    gen_state: RefCell<GeneratorState>,
     /// [`Self::name`] and [`Self::qualname`] when they aren't the code's,
     /// [`Self::origin`] and [`Self::finalizer`]: the state few generators
     /// carry, allocated when first set.
@@ -5530,6 +5531,13 @@ pub struct PyGenerator {
     /// suspends, which tracks it (see `Interpreter::park_suspended_boxed`).
     /// One that finishes first is never tracked.
     pub track_deferred: crate::sync::Cell<bool>,
+    /// The collapsed `yield from` chain this generator is a level of (the
+    /// address of its `ChainRec`, see `Interpreter::core_send_collapse`),
+    /// or 0. A level of a parked chain is in
+    /// [`GeneratorState::Delegating`] although nothing runs it:
+    /// [`Self::state`] puts the chain back as running it would have left
+    /// it before anything sees the state.
+    pub(crate) chain: crate::sync::Cell<usize>,
 }
 
 /// The state of a [`PyGenerator`] that few carry.
@@ -5666,15 +5674,36 @@ impl PyGenerator {
         Self {
             kind,
             code,
-            state: RefCell::new(GeneratorState::Created(frame)),
+            gen_state: RefCell::new(GeneratorState::Created(frame)),
             extras: RefCell::new(None),
             hooks_inited: crate::sync::Cell::new(false),
             finalize_ran: crate::sync::Cell::new(false),
             gc_slot: crate::gc_trace::GcSlot::new(),
             track_deferred: crate::sync::Cell::new(false),
+            chain: crate::sync::Cell::new(0),
         }
     }
 
+    /// The generator's state. A level of a parked `yield from` chain (see
+    /// [`Self::chain`]) is first suspended in its `yield from`, with the
+    /// rest of the chain, as resuming the chain level by level leaves it.
+    #[inline(always)]
+    pub fn state(&self) -> &RefCell<GeneratorState> {
+        let chain = self.chain.get();
+        if chain != 0 {
+            crate::settle_parked_chain(chain);
+        }
+        &self.gen_state
+    }
+
+    /// The state as it is, a parked chain's level still `Delegating`: for
+    /// the chain machinery, and for reads that treat a delegating frame
+    /// as a suspended one (the collector's traversal).
+    #[inline(always)]
+    pub(crate) fn state_raw(&self) -> &RefCell<GeneratorState> {
+        &self.gen_state
+    }
+
     /// [`Self::new`] with the name objects already built (a function's
     /// pinned `__name__`/`__qualname__`, shared rather than copied).
     pub fn with_names(
@@ -5697,7 +5726,7 @@ impl PyGenerator {
     }
 
     pub fn is_finished(&self) -> bool {
-        matches!(&*self.state.borrow(), GeneratorState::Finished)
+        matches!(&*self.state().borrow(), GeneratorState::Finished)
     }
 
     /// Has the body run at all? A generator still in `Created` has
@@ -5705,7 +5734,7 @@ impl PyGenerator {
     /// `gen_close` marks such a generator closed without throwing
     /// `GeneratorExit` at it, and `gen_dealloc` frees it outright.
     pub fn is_unstarted(&self) -> bool {
-        self.state
+        self.state()
             .try_borrow()
             .is_ok_and(|s| matches!(&*s, GeneratorState::Created(_)))
     }
@@ -5726,8 +5755,15 @@ impl Drop for PyGenerator {
         // pipeline (`chain(chain(chain(…)))`); dropping it inline recurses
         // one native stack frame per link and overflows on long chains, so
         // route it through the iterative trampoline below.
+        // A parked chain's level settles first (its levels are alive: each
+        // is held by the one above, the outermost by this generator's
+        // owners, and this one's frame is still here).
+        let chain = self.chain.get();
+        if chain != 0 {
+            crate::settle_parked_chain(chain);
+        }
         // (Exclusive: no borrow of the state can be live.)
-        let state = self.state.get_mut();
+        let state = self.gen_state.get_mut();
         if matches!(
             state,
             GeneratorState::Suspended(_) | GeneratorState::Created(_) | GeneratorState::Delegating(_)
diff --git a/crates/weavepy-vm/src/stdlib/gc_real.rs b/crates/weavepy-vm/src/stdlib/gc_real.rs
index 3c8fe90d..968de450 100644
--- a/crates/weavepy-vm/src/stdlib/gc_real.rs
+++ b/crates/weavepy-vm/src/stdlib/gc_real.rs
@@ -402,7 +402,7 @@ fn referrer_edges(obj: &Object, visit: &mut dyn FnMut(&Object)) {
         // stack rather than in the generator's state box.
         Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g)
             if matches!(
-                g.state.try_borrow().as_deref(),
+                g.state().try_borrow().as_deref(),
                 Ok(crate::object::GeneratorState::Running)
             ) =>
         {
diff --git a/crates/weavepy-vm/src/vm_singletons.rs b/crates/weavepy-vm/src/vm_singletons.rs
index fe8b3f1d..a23bb32c 100644
--- a/crates/weavepy-vm/src/vm_singletons.rs
+++ b/crates/weavepy-vm/src/vm_singletons.rs
@@ -160,9 +160,9 @@ pub(crate) fn finalize_on_last_release<T: ?Sized + 'static>(arc: std::sync::Arc<
         // SAFETY: a read that runs no code (`peek_mut` rejects a live
         // borrow or shared cells, which take the borrow).
         let owes = !g.finalize_ran.get()
-            && match unsafe { g.state.peek_mut() } {
+            && match unsafe { g.state().peek_mut() } {
                 Some(state) => owes_for(state),
-                None => g.state.try_borrow().is_ok_and(|state| owes_for(&state)),
+                None => g.state().try_borrow().is_ok_and(|state| owes_for(&state)),
             };
         // A plain generator suspended outside any handler of its own
         // (CPython's `gen_close` shortcut) just finishes, here: nothing
diff --git a/tests/regrtest/test_yield_from_parked_chains.py b/tests/regrtest/test_yield_from_parked_chains.py
new file mode 100644
index 00000000..ea33623f
--- /dev/null
+++ b/tests/regrtest/test_yield_from_parked_chains.py
@@ -0,0 +1,412 @@
+"""Observing and changing the levels of a `yield from` chain between items.
+
+Once a chain has passed a value up, its levels may stay collapsed: the next
+resume of the outermost level runs the innermost generator directly again.
+Anything that reaches an intermediate level in between (its state, frame,
+locals, `send`, `throw`, `close`, its release, a tracer) must find it just
+as resuming the chain level by level leaves it: suspended in its
+`yield from`, with every later item, exception and finalizer unchanged.
+"""
+
+import asyncio
+import gc
+import inspect
+import sys
+import traceback
+import types
+import unittest
+
+N = 50
+
+
+def leaf(n, log=None):
+    try:
+        for i in range(n):
+            yield i
+    finally:
+        if log is not None:
+            log.append("leaf")
+
+
+def level(depth, n, log=None, gens=None):
+    try:
+        if depth == 0:
+            inner = leaf(n, log)
+        else:
+            inner = level(depth - 1, n, log, gens)
+        if gens is not None:
+            gens.append(inner)
+        result = yield from inner
+        return result
+    finally:
+        if log is not None:
+            log.append(depth)
+
+
+def started(depth, n=N, items=3, log=None):
+    """A chain `depth + 1` levels deep, `items` items in, with its levels
+    outermost first (the innermost generator last)."""
+    gens = []
+    top = level(depth, n, log, gens)
+    for i in range(items):
+        assert next(top) == i
+    return top, [top] + gens
+
+
+class ParkedChainTest(unittest.TestCase):
+    def test_levels_suspended(self):
+        top, gens = started(5)
+        for g in gens:
+            self.assertFalse(g.gi_running)
+            self.assertTrue(g.gi_suspended)
+            self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_SUSPENDED)
+        for outer, inner in zip(gens, gens[1:]):
+            self.assertIs(outer.gi_yieldfrom, inner)
+        self.assertIsNone(gens[-1].gi_yieldfrom)
+        self.assertEqual(list(top), list(range(3, N)))
+
+    def test_frames_between_items(self):
+        top, gens = started(5)
+        line = level.__code__.co_firstlineno + 8
+        for g in gens[:-1]:
+            frame = g.gi_frame
+            self.assertEqual(frame.f_code.co_name, "level")
+            self.assertEqual(frame.f_lineno, line)
+            self.assertIsNone(frame.f_back)
+        frames = [g.gi_frame for g in gens[:-1]]
+        self.assertEqual(len({f.f_lasti for f in frames}), 1)
+        self.assertEqual(next(top), 3)
+        # Still the same frames, on the same line.
+        self.assertEqual([g.gi_frame for g in gens[:-1]], frames)
+        self.assertEqual({f.f_lineno for f in frames}, {line})
+        self.assertEqual(list(top), list(range(4, N)))
+
+    def test_locals_of_a_level(self):
+        top, gens = started(4)
+        mid = gens[2]
+        self.assertEqual(mid.gi_frame.f_locals["depth"], 2)
+        self.assertIs(mid.gi_frame.f_locals["inner"], gens[3])
+        self.assertEqual(next(top), 3)
+        self.assertEqual(mid.gi_frame.f_locals["depth"], 2)
+        self.assertEqual(list(top), list(range(4, N)))
+
+    def test_next_on_a_level(self):
+        top, gens = started(5)
+        mid = gens[3]
+        self.assertEqual(next(mid), 3)
+        self.assertEqual(next(mid), 4)
+        self.assertEqual(next(top), 5)
+        self.assertEqual(next(gens[1]), 6)
+        self.assertEqual(next(top), 7)
+        self.assertEqual(list(top), list(range(8, N)))
+
+    def test_next_on_the_innermost(self):
+        top, gens = started(5)
+        self.assertEqual(next(gens[-1]), 3)
+        self.assertEqual(next(top), 4)
+        self.assertEqual(next(gens[-1]), 5)
+        self.assertEqual(list(top), list(range(6, N)))
+
+    def test_send_to_a_level(self):
+        def echo():
+            got = None
+            while True:
+                got = yield got
+
+        def wrap(g):
+            return (yield from g)
+
+        inner = echo()
+        gens = [inner]
+        for _ in range(5):
+            gens.insert(0, wrap(gens[0]))
+        top = gens[0]
+        next(top)
+        for i in range(3):
+            self.assertEqual(top.send(i), i)
+        self.assertEqual(gens[2].send("mid"), "mid")
+        self.assertEqual(next(top), None)
+        self.assertEqual(top.send(7), 7)
+        self.assertEqual(gens[4].send(8), 8)
+        self.assertEqual(top.send(9), 9)
+
+    def test_throw_into_a_level(self):
+        log = []
+        top, gens = started(5, log=log)
+        with self.assertRaises(KeyError):
+            gens[3].throw(KeyError("x"))
+        # The leaf and two levels finished, innermost first.
+        self.assertEqual(log, ["leaf", 0, 1, 2])
+        self.assertEqual(inspect.getgeneratorstate(gens[3]), inspect.GEN_CLOSED)
+        self.assertEqual(inspect.getgeneratorstate(gens[2]), inspect.GEN_SUSPENDED)
+        # The level above finds its finished delegate exhausted, and the
+        # rest of the chain finishes in turn.
+        with self.assertRaises(StopIteration):
+            next(top)
+        self.assertEqual(log, ["leaf", 0, 1, 2, 3, 4, 5])
+
+    def test_throw_caught_by_a_level(self):
+        def catching(g):
+            try:
+                yield from g
+            except KeyError:
+                yield "caught"
+            yield "after"
+
+        inner = leaf(N)
+        mid = catching(inner)
+        top = catching(catching(mid))
+        self.assertEqual([next(top) for _ in range(3)], [0, 1, 2])
+        self.assertEqual(mid.throw(KeyError), "caught")
+        self.assertEqual(inspect.getgeneratorstate(inner), inspect.GEN_CLOSED)
+        self.assertEqual(next(top), "after")
+        self.assertEqual(next(top), "after")
+        self.assertEqual(next(top), "after")
+        with self.assertRaises(StopIteration):
+            next(top)
+
+    def test_close_a_level(self):
+        log = []
+        top, gens = started(5, log=log)
+        gens[3].close()
+        self.assertEqual(log, ["leaf", 0, 1, 2])
+        # The level above finds its delegate exhausted (its return value
+        # `None`) and finishes in turn, as do the rest.
+        with self.assertRaises(StopIteration):
+            next(top)
+        self.assertEqual(log, ["leaf", 0, 1, 2, 3, 4, 5])
+
+    def test_close_the_top(self):
+        log = []
+        top, gens = started(5, log=log)
+        top.close()
+        self.assertEqual(log, ["leaf", 0, 1, 2, 3, 4, 5])
+        for g in gens:
+            self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_CLOSED)
+
+    def test_release_runs_finalizers_at_once(self):
+        log = []
+        top, gens = started(5, log=log)
+        del gens
+        self.assertEqual(log, [])
+        del top
+        self.assertEqual(log, ["leaf", 0, 1, 2, 3, 4, 5])
+
+    def test_release_of_a_shared_level(self):
+        log = []
+        top, gens = started(5, log=log)
+        mid = gens[2]
+        del gens
+        del top
+        # The levels above `mid` finish at their `yield from`, which
+        # closes `mid`: everything finishes.
+        self.assertEqual(log, ["leaf", 0, 1, 2, 3, 4, 5])
+        self.assertEqual(inspect.getgeneratorstate(mid), inspect.GEN_CLOSED)
+
+    def test_collect_between_items(self):
+        top, gens = started(5)
+        gc.collect()
+        self.assertTrue(all(g.gi_suspended for g in gens))
+        self.assertEqual(next(top), 3)
+        gc.collect()
+        self.assertEqual(list(top), list(range(4, N)))
+
+    def test_cycle_through_a_level(self):
+        log = []
+        top, gens = started(5, log=log)
+        holder = []
+        holder.append(gens[2])
+        # A cycle: the innermost level's local refers back to the chain.
+        gens[-1].gi_frame.f_locals  # (a read only)
+        cycle = [top, holder]
+        holder.append(cycle)
+        del top, gens, holder, cycle
+        gc.collect()
+        self.assertEqual(log, ["leaf", 0, 1, 2, 3, 4, 5])
+
+    def test_raise_after_parking(self):
+        def failing(n):
+            for i in range(n):
+                yield i
+            raise ValueError("deep")
+
+        def wrap(depth):
+            if depth == 0:
+                yield from failing(5)
+            else:
+                yield from wrap(depth - 1)
+
+        g = wrap(4)
+        out = []
+        try:
+            for x in g:
+                out.append(x)
+        except ValueError as e:
+            names = [f.name for f in traceback.extract_tb(e.__traceback__)]
+        else:
+            self.fail("no ValueError")
+        self.assertEqual(out, list(range(5)))
+        self.assertEqual(names, ["test_raise_after_parking"] + ["wrap"] * 5 + ["failing"])
+
+    def test_stop_iteration_values(self):
+        def counting(n):
+            for i in range(n):
+                yield i
+            return n
+
+        def adding(depth, n):
+            if depth == 0:
+                got = yield from counting(n)
+            else:
+                got = yield from adding(depth - 1, n)
+            yield ("level", depth, got)
+            return got + 1
+
+        out = list(adding(4, 6))
+        self.assertEqual(out[:6], list(range(6)))
+        self.assertEqual(
+            out[6:],
+            [("level", 0, 6), ("level", 1, 7), ("level", 2, 8), ("level", 3, 9), ("level", 4, 10)],
+        )
+
+    def test_two_consumers_share_a_chain(self):
+        def wrap(g):
+            yield from g
+
+        shared = level(4, N)
+        a = wrap(shared)
+        b = wrap(shared)
+        got = []
+        for _ in range(10):
+            got.append(next(a))
+            got.append(next(b))
+        self.assertEqual(got, list(range(20)))
+        self.assertEqual(list(a), list(range(20, N)))
+        self.assertEqual(list(b), [])
+
+    def test_trace_after_parking(self):
+        top, gens = started(4)
+        events = []
+
+        def tracer(frame, event, arg):
+            if event in ("call", "return"):
+                events.append((event, frame.f_code.co_name))
+            return tracer
+
+        sys.settrace(tracer)
+        try:
+            value = next(top)
+        finally:
+            sys.settrace(None)
+        self.assertEqual(value, 3)
+        self.assertEqual(
+            events,
+            [("call", "level")] * 5
+            + [("call", "leaf"), ("return", "leaf")]
+            + [("return", "level")] * 5,
+        )
+        self.assertEqual(list(top), list(range(4, N)))
+
+    def test_generator_expression_consumer(self):
+        top, gens = started(6)
+        self.assertEqual(sum(x for x in top if x % 2), sum(i for i in range(3, N) if i % 2))
+
+    def test_coroutine_chain_between_sends(self):
+        @types.coroutine
+        def tick(i):
+            return (yield i)
+
+        async def leaf_coro(n):
+            total = 0
+            for i in range(n):
+                total += await tick(i)
+            return total
+
+        async def wrap(depth, n, coros):
+            if depth == 0:
+                inner = leaf_coro(n)
+            else:
+                inner = wrap(depth - 1, n, coros)
+            coros.append(inner)
+            return await inner + 1
+
+        coros = []
+        coro = wrap(5, N, coros)
+        self.assertEqual(coro.send(None), 0)
+        self.assertEqual(coro.send(1), 1)
+        mid = coros[-3]
+        self.assertEqual(inspect.getcoroutinestate(mid), inspect.CORO_SUSPENDED)
+        self.assertIsNotNone(mid.cr_await)
+        self.assertEqual(mid.cr_frame.f_code.co_name, "wrap")
+        value = 2
+        try:
+            while True:
+                value = coro.send(1)
+        except StopIteration as stop:
+            result = stop.value
+        self.assertEqual(value, N - 1)
+        self.assertEqual(result, N + 6)
+
+    def test_asyncio_cancel_nested(self):
+        log = []
+
+        async def inner():
+            try:
+                while True:
+                    await asyncio.sleep(0)
+            finally:
+                log.append("inner")
+
+        async def wrap(depth):
+            try:
+                if depth == 0:
+                    await inner()
+                else:
+                    await wrap(depth - 1)
+            finally:
+                log.append(depth)
+
+        async def main():
+            task = asyncio.ensure_future(wrap(5))
+            for _ in range(20):
+                await asyncio.sleep(0)
+            task.cancel()
+            with self.assertRaises(asyncio.CancelledError):
+                await task
+            stack = []
+            return stack
+
+        asyncio.run(main())
+        self.assertEqual(log, ["inner", 0, 1, 2, 3, 4, 5])
+
+    def test_asyncio_get_stack(self):
+        async def inner(ready):
+            ready.set_result(None)
+            await asyncio.sleep(3600)
+
+        async def wrap(depth, ready):
+            if depth == 0:
+                await inner(ready)
+            else:
+                await wrap(depth - 1, ready)
+
+        async def main():
+            loop = asyncio.get_running_loop()
+            ready = loop.create_future()
+            task = asyncio.ensure_future(wrap(4, ready))
+            await ready
+            for _ in range(5):
+                await asyncio.sleep(0)
+            names = [f.f_code.co_name for f in task.get_stack(limit=100)]
+            task.cancel()
+            try:
+                await task
+            except asyncio.CancelledError:
+                pass
+            return names
+
+        self.assertEqual(asyncio.run(main())[:1], ["wrap"])
+
+
+if __name__ == "__main__":
+    unittest.main()

From 469ac4c6663257149dd79a87ce8c1573bbae713a Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 20:40:06 -0700
Subject: [PATCH 130/173] perf: step a simple generator in place for next() and
 native consumers

`next(gen)` and `next(gen, default)` in compiled code resumed the
generator as a full inline activation even when its body could take a
fast step (the core loop's own `next()` arm and every `FOR_ITER` already
tried one first), and a native consumer (`enumerate`, `zip`, `map`,
`__next__`) always went through the lean resume's frame setup. Both now
try `gen_fast_next` first, on the same terms as the paths that already
use it; anything it declines takes the old route.

Measured (instructions retired per item, macOS):
- `t += next(g)` in a compiled loop: 1679 -> 1005.
- `for i, x in enumerate(gen(n))`: 2542 -> 2297.
- Suite: `nqueens` 1.81x -> 1.80x, `graph_search` 1.22x -> 1.21x CPython.
---
 crates/weavepy-vm/src/frame_jit.rs            |  53 +++++-
 crates/weavepy-vm/src/lib.rs                  |  38 ++++-
 .../test_generator_next_fast_steps.py         | 154 ++++++++++++++++++
 3 files changed, 242 insertions(+), 3 deletions(-)
 create mode 100644 tests/regrtest/test_generator_next_fast_steps.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 6b322228..c223dc40 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -5138,9 +5138,35 @@ unsafe extern "C" fn h_call(
             if st.sw.is_null() {
                 return CALL_DECLINED;
             }
+            let interp = &mut *st.interp.cast_mut();
+            // A simple body steps to its next yield in place, as the core
+            // loop's arm steps it (see `gen_fast`).
+            if let (Object::Generator(g), true) =
+                (&*base.add(start + 2), Interpreter::core_drops_plain())
+            {
+                match interp.gen_fast_next(g, st.snap_gen, 0, st.maybe_dead) {
+                    crate::gen_fast::GenNext::Yielded(v) => {
+                        // The builtin and the generator (the stack's own
+                        // slots hold them past the step) leave by plain
+                        // decrements; the value takes the callee's slot.
+                        crate::release_run(base.add(start), 3);
+                        base.add(start).write(v);
+                        st.len = start + 1;
+                        return 0;
+                    }
+                    // `next(gen)` raises `StopIteration(value)`.
+                    crate::gen_fast::GenNext::Exhausted(v) => {
+                        crate::release_run(base.add(start), 3);
+                        st.len = start;
+                        st.pc = pc + 1;
+                        st.err = Some(crate::error::stop_iteration_with(v));
+                        return RAISED;
+                    }
+                    crate::gen_fast::GenNext::Declined | crate::gen_fast::GenNext::Partial => {}
+                }
+            }
             sync(st);
             let sw = &mut *st.sw;
-            let interp = &mut *st.interp.cast_mut();
             if !interp.core_gen_resume(sw, pc, crate::InlineResume::NextCall) {
                 sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
             }
@@ -5158,9 +5184,32 @@ unsafe extern "C" fn h_call(
             if st.sw.is_null() {
                 return CALL_DECLINED;
             }
+            let interp = &mut *st.interp.cast_mut();
+            // A simple body's step in place, as for `next(gen)` above.
+            if let (Object::Generator(g), true) =
+                (&*base.add(start + 2), Interpreter::core_drops_plain())
+            {
+                match interp.gen_fast_next(g, st.snap_gen, 0, st.maybe_dead) {
+                    crate::gen_fast::GenNext::Yielded(v) => {
+                        crate::release_run(base.add(start), 4);
+                        base.add(start).write(v);
+                        st.len = start + 1;
+                        return 0;
+                    }
+                    // The default is the result.
+                    crate::gen_fast::GenNext::Exhausted(v) => {
+                        drop(v);
+                        let default = base.add(start + 3).read();
+                        crate::release_run(base.add(start), 3);
+                        base.add(start).write(default);
+                        st.len = start + 1;
+                        return 0;
+                    }
+                    crate::gen_fast::GenNext::Declined | crate::gen_fast::GenNext::Partial => {}
+                }
+            }
             sync(st);
             let sw = &mut *st.sw;
-            let interp = &mut *st.interp.cast_mut();
             if !interp.core_gen_resume(sw, pc, crate::InlineResume::NextDefault) {
                 sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
             }
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 58b310d0..773f6990 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -19274,7 +19274,7 @@ impl Interpreter {
     /// no reference tracing), so a droppable value leaves by a plain
     /// decrement (see [`Self::core_droppable`]).
     #[inline(always)]
-    fn core_drops_plain() -> bool {
+    pub(crate) fn core_drops_plain() -> bool {
         // (The dict and function watchers' and the reftrace printer's
         // flags, tested with one load.)
         crate::sync::release_gates() & crate::sync::RELEASE_OBSERVER_GATES == 0
@@ -44425,6 +44425,30 @@ impl Interpreter {
     /// on the frame or its cached shell), it suspended outside any
     /// handler (no detached exception state to restore), and its code
     /// is the interpreter's — else `None` and the general path runs.
+    /// [`Self::generator_send_fold`]'s resume of `gen` with `None` by fast
+    /// step (see `gen_fast`), on the lean resume's terms: `None` when the
+    /// generator or its body declines, with nothing run (or the body
+    /// stopped partway, parked for the lean resume to go on with).
+    #[inline]
+    fn generator_next_fast(
+        &mut self,
+        gen: &Rc<PyGenerator>,
+        raw_return: bool,
+    ) -> Option<Result<Object, RuntimeError>> {
+        let snap_gen = self.lean_snapshot()?;
+        if self.dbg_sample
+            || crate::trace::eval_frame_record_active()
+            || crate::stdlib::greenlet_native::on_greenlet_stack()
+        {
+            return None;
+        }
+        match self.gen_fast_next(gen, snap_gen, 0, crate::gc_trace::maybe_dead_flag()) {
+            gen_fast::GenNext::Yielded(v) => Some(Ok(v)),
+            gen_fast::GenNext::Exhausted(v) => Some(Self::generator_returned(v, raw_return)),
+            gen_fast::GenNext::Declined | gen_fast::GenNext::Partial => None,
+        }
+    }
+
     fn generator_send_lean(
         &mut self,
         gen: &Rc<PyGenerator>,
@@ -44768,6 +44792,18 @@ impl Interpreter {
         // Consumed here, so a generator this one resumes in turn raises
         // `StopIteration` as usual.
         let raw_return = GEN_RAW_RETURN.with(|r| r.replace(false));
+        // A simple body's resume steps to its next yield in place, as a
+        // `FOR_ITER` over it does (see `gen_fast`): no activation at all.
+        // (A draining consumer's fold takes the lean resume, which folds
+        // every yield it can in one step.)
+        if fold.is_none()
+            && matches!(sent, Object::None)
+            && gen.kind == crate::object::CoroutineKind::Generator
+        {
+            if let Some(r) = self.generator_next_fast(gen, raw_return) {
+                return r;
+            }
+        }
         if let Some(r) = self.generator_send_lean(gen, &sent, fold, raw_return) {
             return r;
         }
diff --git a/tests/regrtest/test_generator_next_fast_steps.py b/tests/regrtest/test_generator_next_fast_steps.py
new file mode 100644
index 00000000..f64354da
--- /dev/null
+++ b/tests/regrtest/test_generator_next_fast_steps.py
@@ -0,0 +1,154 @@
+"""`next()` and native consumers stepping simple generators in place.
+
+A simple generator body may run to its next yield without an activation
+of its own, whether the consumer is a `FOR_ITER`, a `next()` call from
+compiled or interpreted code, or a native consumer (`enumerate`, `zip`,
+`map`, `iter` with a sentinel). Values, exhaustion (`StopIteration` and
+its value, a default), the generator's state and the release of what it
+yields must not depend on which consumer stepped it.
+"""
+
+import inspect
+import unittest
+
+ROUNDS = 300
+
+
+def counting(n):
+    for i in range(n):
+        yield i
+    return n * 10
+
+
+class Noted:
+    log = []
+
+    def __init__(self, i):
+        self.i = i
+
+    def __del__(self):
+        Noted.log.append(self.i)
+
+
+def noted(n):
+    for i in range(n):
+        yield Noted(i)
+
+
+class NextFastStepsTest(unittest.TestCase):
+    def test_next_until_exhausted(self):
+        def drain(g):
+            out = []
+            while True:
+                try:
+                    out.append(next(g))
+                except StopIteration as e:
+                    return out, e.value
+
+        for _ in range(ROUNDS):
+            out, value = drain(counting(5))
+            self.assertEqual(out, [0, 1, 2, 3, 4])
+            self.assertEqual(value, 50)
+
+    def test_next_with_default(self):
+        marker = object()
+
+        def drain(g):
+            out = []
+            while (x := next(g, marker)) is not marker:
+                out.append(x)
+            return out
+
+        for _ in range(ROUNDS):
+            g = counting(4)
+            self.assertEqual(drain(g), [0, 1, 2, 3])
+            self.assertIs(next(g, marker), marker)
+            self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_CLOSED)
+
+    def test_next_state_between_items(self):
+        def step(g, n):
+            for _ in range(n):
+                next(g)
+
+        for _ in range(ROUNDS):
+            g = counting(10)
+            self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_CREATED)
+            step(g, 3)
+            self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_SUSPENDED)
+            self.assertEqual(g.gi_frame.f_locals["i"], 2)
+            self.assertEqual(next(g), 3)
+            self.assertEqual(list(g), [4, 5, 6, 7, 8, 9])
+
+    def test_release_of_yielded_values(self):
+        def take(g, n):
+            for _ in range(n):
+                x = next(g)
+                del x
+
+        for _ in range(ROUNDS // 10):
+            Noted.log = []
+            g = noted(5)
+            take(g, 3)
+            self.assertEqual(Noted.log, [0, 1, 2])
+            del g
+            self.assertEqual(Noted.log, [0, 1, 2])
+
+    def test_release_of_the_default(self):
+        def last(g, default):
+            x = next(g, default)
+            return x
+
+        for _ in range(ROUNDS // 10):
+            Noted.log = []
+            g = counting(0)
+            self.assertIsInstance(last(g, Noted(-1)), Noted)
+            self.assertEqual(Noted.log, [-1])
+
+    def test_native_consumers(self):
+        for _ in range(ROUNDS):
+            self.assertEqual(list(enumerate(counting(3))), [(0, 0), (1, 1), (2, 2)])
+            self.assertEqual(list(zip(counting(3), counting(4))), [(0, 0), (1, 1), (2, 2)])
+            self.assertEqual(list(map(next, [counting(2)] * 2)), [0, 1])
+            g = counting(3)
+            self.assertEqual(list(map(next, [g, g, g])), [0, 1, 2])
+            self.assertEqual(list(iter(counting(9).__next__, 4)), [0, 1, 2, 3])
+
+    def test_native_consumer_exhaustion_value(self):
+        for _ in range(ROUNDS):
+            g = counting(2)
+            self.assertEqual(list(enumerate(g)), [(0, 0), (1, 1)])
+            with self.assertRaises(StopIteration) as cm:
+                g.__next__()
+            self.assertIsNone(cm.exception.value)
+            g = counting(1)
+            self.assertEqual(g.__next__(), 0)
+            with self.assertRaises(StopIteration) as cm:
+                g.__next__()
+            self.assertEqual(cm.exception.value, 10)
+
+    def test_running_inside(self):
+        seen = []
+
+        def peeking():
+            for i in range(3):
+                seen.append(me.gi_running)
+                yield i
+
+        for _ in range(ROUNDS // 10):
+            seen.clear()
+            me = peeking()
+            self.assertEqual([next(me), next(me), next(me)], [0, 1, 2])
+            self.assertEqual(seen, [True, True, True])
+
+    def test_reentrant_next_raises(self):
+        def selfish():
+            yield next(me)
+
+        for _ in range(ROUNDS // 10):
+            me = selfish()
+            with self.assertRaisesRegex(ValueError, "already executing"):
+                next(me)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 0f30b6e2e07999b5300b42b9393ba998bc45898e Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 21:37:02 -0700
Subject: [PATCH 131/173] perf: collapse a yield-from chain from FOR_ITER and
 next() too

A chain only collapsed below a `SEND`: the generator a `for` loop or
`next()` resumed ran as an activation of its own, and its `SEND` then
resumed the parked levels below it. That generator is now the chain's
outermost level as well, resumed by the consumer's own instruction
(`core_iter_collapse`), and a generator whose delegate heads a parked
chain joins that chain instead of resuming it (`collapse_prepend`), so a
chain built up by `SEND`s merges with the level above once. A `next()`
call's chain keeps its generator alive while it runs (the call's operands
did); the chain lets it go when it parks. `gen_fast_next` declines a
chain's level without settling it.

Measured (instructions retired per item, macOS):
- `for x in` a chain of plain `yield from` levels: depth 2 1629 -> 1246,
  depth 4 1878 -> 1260, depth 16 2048 -> 1323 (CPython: 1014 at depth 4,
  2453 at depth 16).
- Suite `generators_tree`: 1.65x -> 1.53x CPython.
---
 crates/weavepy-vm/src/gen_fast.rs             |   7 +-
 crates/weavepy-vm/src/lib.rs                  | 263 +++++++++++++++---
 .../regrtest/test_yield_from_parked_chains.py |  54 ++++
 3 files changed, 288 insertions(+), 36 deletions(-)

diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index 093070f5..e2b1a313 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -1021,11 +1021,16 @@ impl Interpreter {
         {
             return GenNext::Declined;
         }
+        // A level of a collapsed `yield from` chain is the inline resume's
+        // (which resumes a parked chain without settling it).
+        if g.chain.get() != 0 {
+            return GenNext::Declined;
+        }
         // Validate and take the frame under one exclusive view; no Python
         // runs while it's held.
         // SAFETY: nothing below reaches the cell again until `state`'s
         // last use (`peek_mut` rejects a live guard or shared cells).
-        let Some(state) = (unsafe { g.state().peek_mut() }) else {
+        let Some(state) = (unsafe { g.state_raw().peek_mut() }) else {
             return GenNext::Declined;
         };
         let (GeneratorState::Suspended(boxed) | GeneratorState::Created(boxed)) = &*state else {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 773f6990..3fbd2cc3 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -20878,7 +20878,17 @@ impl Interpreter {
                     }
                 }
             },
-            _ => {
+            // A generator suspended in a `yield from` collapses its chain
+            // as a `SEND` to it would (see `Self::core_iter_collapse`).
+            InlineResume::ForIter | InlineResume::NextCall | InlineResume::NextDefault => {
+                if self.core_iter_collapse(sw, pc, mode) == SendHop::Collapsed {
+                    return true;
+                }
+                if !self.core_gen_resume_one(sw, pc, mode) {
+                    return false;
+                }
+            }
+            InlineResume::SendCall => {
                 if !self.core_gen_resume_one(sw, pc, mode) {
                     return false;
                 }
@@ -20907,6 +20917,66 @@ impl Interpreter {
         true
     }
 
+    /// [`Self::core_gen_resume`]'s `FOR_ITER` or `next()` (`mode`) at `pc`
+    /// of a generator suspended in a `yield from`: the generator becomes
+    /// the outermost level of a collapsed chain, as the delegate of a
+    /// `SEND` does (see [`Self::core_send_collapse`]), or resumes its
+    /// parked one. Anything else is declined, touching nothing.
+    #[inline(never)]
+    fn core_iter_collapse(&mut self, sw: &mut CoreSwitch, pc: usize, mode: InlineResume) -> SendHop {
+        // SAFETY: see `CoreSwitch`: the running activation is synced, the
+        // generator where `mode` puts it (checked by the caller).
+        let frame = unsafe { &*sw.cur };
+        let n = frame.stack.len();
+        let at = match mode {
+            InlineResume::ForIter | InlineResume::NextCall => n.checked_sub(1),
+            _ => n.checked_sub(2),
+        };
+        let Some(Object::Generator(g)) = at.and_then(|at| frame.stack.get(at)) else {
+            return SendHop::Declined;
+        };
+        if g.chain.get() != 0 {
+            return match Self::parked_head(g) {
+                Some(addr) => self.core_send_unpark(sw, pc, mode, addr),
+                None => SendHop::Declined,
+            };
+        }
+        let Some(gf) = Self::collapse_frame(g) else {
+            return SendHop::Declined;
+        };
+        // SAFETY: suspended (checked): a read between instructions.
+        let Some((next, next_send)) = Self::collapse_next(unsafe { &*gf }) else {
+            return SendHop::Declined;
+        };
+        // Its delegate heads a parked chain: the generator joins it.
+        if let Some(addr) = Self::parked_head(next) {
+            // SAFETY: `g` is suspended in a `yield from` of the chain's
+            // outermost level (checked).
+            unsafe { Self::collapse_prepend(addr, Rc::as_ptr(g), gf, next_send) };
+            return self.core_send_unpark(sw, pc, mode, addr);
+        }
+        let Some(next_frame) = Self::collapse_frame(next) else {
+            return SendHop::Declined;
+        };
+        let next: *const Rc<PyGenerator> = next;
+        // SAFETY: `next` lives on `g`'s stack, which nothing changes before
+        // the walk reads it.
+        if self.core_send_collapse(
+            sw,
+            pc,
+            mode,
+            Rc::as_ptr(g),
+            gf,
+            unsafe { &*next },
+            next_send,
+            next_frame,
+        ) {
+            SendHop::Collapsed
+        } else {
+            SendHop::Declined
+        }
+    }
+
     /// One level of [`Self::core_gen_resume`]'s `yield from` chain: the
     /// running generator activation's `SEND` at `send_pc` to the suspended
     /// generator under its sent value, as [`Self::try_inline_gen`] resumes
@@ -20934,13 +21004,8 @@ impl Interpreter {
         let GeneratorState::Suspended(boxed) = &*state else {
             // The outermost level of a parked chain: its innermost
             // generator resumes directly (anything else settles it).
-            let chain = g.chain.get();
-            if chain != 0 && sent_none {
-                // SAFETY: a level's record is alive (see `ChainRec`).
-                let rec = unsafe { &*(chain as *const ChainRec) };
-                if rec.parked && std::ptr::eq(rec.levels[0].gen, Rc::as_ptr(g)) {
-                    return self.core_send_unpark(sw, send_pc, chain);
-                }
+            if let (true, Some(addr)) = (sent_none, Self::parked_head(g)) {
+                return self.core_send_unpark(sw, send_pc, InlineResume::Send, addr);
             }
             return SendHop::Declined;
         };
@@ -20965,11 +21030,32 @@ impl Interpreter {
         if sent_none && !gf.sent_consumed && g.kind != crate::object::CoroutineKind::AsyncGenerator
         {
             if let Some((next, next_send)) = Self::collapse_next(gf) {
+                // Its delegate heads a parked chain: the delegate joins it.
+                if let Some(addr) = Self::parked_head(next) {
+                    // SAFETY: `g` is suspended in a `yield from` of the
+                    // chain's outermost level (checked above).
+                    unsafe {
+                        Self::collapse_prepend(
+                            addr,
+                            Rc::as_ptr(g),
+                            std::ptr::from_ref(gf).cast_mut(),
+                            next_send,
+                        );
+                    }
+                    return self.core_send_unpark(sw, send_pc, InlineResume::Send, addr);
+                }
                 if let Some(next_frame) = Self::collapse_frame(next) {
                     let (g, gf) = (Rc::as_ptr(g), std::ptr::from_ref(gf).cast_mut());
-                    return if self
-                        .core_send_collapse(sw, send_pc, g, gf, next, next_send, next_frame)
-                    {
+                    return if self.core_send_collapse(
+                        sw,
+                        send_pc,
+                        InlineResume::Send,
+                        g,
+                        gf,
+                        next,
+                        next_send,
+                        next_frame,
+                    ) {
                         SendHop::Collapsed
                     } else {
                         SendHop::Declined
@@ -21028,6 +21114,59 @@ impl Interpreter {
         SendHop::Hopped
     }
 
+    /// The address of the parked chain `g` is the outermost level of (see
+    /// [`ChainRec`]), if any.
+    #[inline(always)]
+    fn parked_head(g: &PyGenerator) -> Option<usize> {
+        let addr = g.chain.get();
+        if addr == 0 {
+            return None;
+        }
+        // SAFETY: a level's record is alive (see `ChainRec`).
+        let rec = unsafe { &*(addr as *const ChainRec) };
+        (rec.parked && std::ptr::eq(rec.levels[0].gen, g)).then_some(addr)
+    }
+
+    /// Make `g` (frame `gf`, suspended in the `yield from` at `level_send`
+    /// of the outermost level of the parked chain at `addr`) the chain's
+    /// new outermost level, delegating just past its `SEND`.
+    ///
+    /// # Safety
+    ///
+    /// As described: `g` is alive, qualified as a level (see
+    /// [`Self::collapse_frame`]), and nothing borrows its state.
+    #[cold]
+    #[inline(never)]
+    unsafe fn collapse_prepend(
+        addr: usize,
+        g: *const PyGenerator,
+        gf: *mut Frame,
+        level_send: usize,
+    ) {
+        // SAFETY: the caller's contract; a parked record owns itself.
+        unsafe {
+            let rec = &mut *(addr as *mut ChainRec);
+            let st = (*g).state_raw().as_ptr();
+            let GeneratorState::Suspended(boxed) = std::ptr::read(st) else {
+                unreachable!("checked by the caller");
+            };
+            std::ptr::write(st, GeneratorState::Delegating(boxed));
+            (*g).chain.set(addr);
+            (*gf).pc = level_send as u32 + 1;
+            rec.levels.insert(
+                0,
+                Delegate {
+                    gen: g,
+                    act: LeanAct {
+                        frame: gf,
+                        shell: None,
+                    },
+                    send_pc: level_send,
+                },
+            );
+        }
+    }
+
     /// `g` as a collapsed chain's level or innermost generator (see
     /// [`Self::core_send_collapse`]): suspended, with nothing a lean resume
     /// declines (the checks of [`Self::core_send_hop`]). Its frame, or
@@ -21131,6 +21270,7 @@ impl Interpreter {
         &mut self,
         sw: &mut CoreSwitch,
         send_pc: usize,
+        mode: InlineResume,
         first: *const PyGenerator,
         first_frame: *mut Frame,
         inner: &Rc<PyGenerator>,
@@ -21150,7 +21290,7 @@ impl Interpreter {
                 inner_frame,
             )
         };
-        match self.collapse_commit(sw, send_pc, rec, inner, inner_frame) {
+        match self.collapse_commit(sw, send_pc, mode, rec, inner, inner_frame) {
             Ok(()) => true,
             Err(mut rec) => {
                 for level in rec.levels.drain(..).rev() {
@@ -21231,13 +21371,15 @@ impl Interpreter {
 
     /// Run `inner` (frame `inner_frame`, suspended and qualified), the
     /// innermost generator of the chain whose delegating levels `rec`
-    /// holds, as the delegate of the running activation's `SEND` of `None`
-    /// at `send_pc` (see [`Self::core_send_collapse`]). Past the recursion
-    /// limit, `rec` comes back untouched.
+    /// holds, for the running activation's resume of the outermost level
+    /// at `pc`: a `SEND` of `None` (see [`Self::core_send_collapse`]), a
+    /// `FOR_ITER` or a `next()` (`mode`, as [`Self::try_inline_gen`]
+    /// takes them). Past the recursion limit, `rec` comes back untouched.
     fn collapse_commit(
         &mut self,
         sw: &mut CoreSwitch,
-        send_pc: usize,
+        pc: usize,
+        mode: InlineResume,
         mut rec: Box<ChainRec>,
         inner: *const PyGenerator,
         inner_frame: *mut Frame,
@@ -21274,13 +21416,42 @@ impl Interpreter {
         debug_assert!(std::ptr::eq::<Frame>(&*boxed, inner_frame));
         boxed.gen_first_resume = false;
         // SAFETY: see `CoreSwitch`: the running activation is synced, its
-        // stack `[.., first, None]` (checked by `core_send_hop`).
+        // stack as `mode` has it (checked by the caller).
         let frame = unsafe { &mut *sw.cur };
-        let jump = frame.code.instructions[send_pc].arg;
-        // The sent `None` goes in.
-        frame.stack.pop();
+        // The consumer's operands leave as `inline_gen_commit` takes them;
+        // the outermost level's activation keeps a `next()`'s generator.
+        let arg = match mode {
+            InlineResume::Send => {
+                // The sent `None`.
+                frame.stack.pop();
+                frame.code.instructions[pc].arg | GEN_SEND
+            }
+            InlineResume::ForIter => frame.code.instructions[pc].arg,
+            InlineResume::NextCall => {
+                let (g, b, a) = (frame.stack.pop(), frame.stack.pop(), frame.stack.pop());
+                let Some(Object::Generator(g)) = g else {
+                    unreachable!("checked by `core_iter_collapse`");
+                };
+                rec.hold = Some(g);
+                drop((a, b));
+                GEN_NEXT_CALL
+            }
+            InlineResume::NextDefault => {
+                let default = frame.stack.pop().expect("checked above");
+                let (g, b, a) = (frame.stack.pop(), frame.stack.pop(), frame.stack.pop());
+                let Some(Object::Generator(g)) = g else {
+                    unreachable!("checked by `core_iter_collapse`");
+                };
+                rec.hold = Some(g);
+                frame.stack.push(default);
+                drop((a, b));
+                GEN_NEXT_DEFAULT
+            }
+            InlineResume::SendCall => unreachable!("a send call never collapses"),
+        };
+        // The resume's `None` goes in.
         boxed.stack.push(Object::None);
-        frame.pc = send_pc as u32 + 1;
+        frame.pc = pc as u32 + 1;
         let gen_frame: *mut Frame = &raw mut *boxed;
         let mut act = self.inline_slot();
         debug_assert!(act.gen.is_none() && act.gen_box.is_none() && act.guard.is_none());
@@ -21292,17 +21463,23 @@ impl Interpreter {
             std::ptr::write(&raw mut act.guard, Some(guard));
         }
         act.gen_frame = gen_frame;
-        act.exhaust_arg = jump | GEN_SEND;
+        act.exhaust_arg = arg;
         act.act.frame = gen_frame;
-        act.call_pc = send_pc;
-        act.caller_pending = self.core_pending_enter(sw, frame, send_pc);
+        act.call_pc = pc;
+        act.caller_pending = self.core_pending_enter(sw, frame, pc);
         act.exc_depth = self.exc_info_len();
         // The levels wait on the pending list, outermost first, as each
         // would while the next one ran (the buffer stays put from here).
         rec.pend_at = self.lean_pending.len();
         self.lean_pending.reserve(levels);
-        for level in &mut rec.levels {
-            self.lean_pending.push(&raw mut level.act as usize);
+        // SAFETY: reserved: each entry is written in place, and the length
+        // covers them once all are.
+        unsafe {
+            let base = self.lean_pending.as_mut_ptr().add(rec.pend_at);
+            for (k, level) in rec.levels.iter_mut().enumerate() {
+                base.add(k).write(&raw mut level.act as usize);
+            }
+            self.lean_pending.set_len(rec.pend_at + levels);
         }
         act.chain = Some(rec);
         // SAFETY: see `CoreSwitch`.
@@ -21313,15 +21490,22 @@ impl Interpreter {
         Ok(())
     }
 
-    /// The `SEND` of `None` at `send_pc` of `sw`'s (synced) running
-    /// activation to the outermost level of the parked chain whose record
-    /// is at `addr` (see [`ChainRec`]): its innermost generator resumes
-    /// directly, as [`Self::core_send_collapse`] resumes it, with any
-    /// generators that innermost one has since started delegating to
-    /// added as levels. Anything that doesn't qualify leaves the chain
-    /// parked (the `SEND`'s full handler settles it).
+    /// The `SEND` of `None` (or the `FOR_ITER` or `next()`, `mode`) at
+    /// `pc` of `sw`'s (synced) running activation to the outermost level
+    /// of the parked chain whose record is at `addr` (see [`ChainRec`]):
+    /// its innermost generator resumes directly, as
+    /// [`Self::core_send_collapse`] resumes it, with any generators that
+    /// innermost one has since started delegating to added as levels.
+    /// Anything that doesn't qualify leaves the chain parked (the
+    /// instruction's full handler settles it).
     #[inline(never)]
-    fn core_send_unpark(&mut self, sw: &mut CoreSwitch, send_pc: usize, addr: usize) -> SendHop {
+    fn core_send_unpark(
+        &mut self,
+        sw: &mut CoreSwitch,
+        pc: usize,
+        mode: InlineResume,
+        addr: usize,
+    ) -> SendHop {
         // SAFETY: a parked record owns itself (see `ChainRec`); taken
         // here, put back below unless the chain runs.
         let mut rec = unsafe { Box::from_raw(addr as *mut ChainRec) };
@@ -21361,7 +21545,7 @@ impl Interpreter {
             None => (inner, inner_frame),
         };
         rec.parked = false;
-        match self.collapse_commit(sw, send_pc, rec, inner, inner_frame) {
+        match self.collapse_commit(sw, pc, mode, rec, inner, inner_frame) {
             Ok(()) => SendHop::Collapsed,
             Err(mut rec) => {
                 for level in rec.levels.drain(base..).rev() {
@@ -21387,7 +21571,10 @@ impl Interpreter {
         self.lean_pending.truncate(rec.pend_at);
         crate::recursion::leave_n(crate::recursion::depth_cell(), rec.levels.len());
         rec.parked = true;
+        let hold = rec.hold.take();
         let _ = Box::into_raw(rec);
+        // (Last: its release may settle the chain.)
+        drop(hold);
     }
 
     /// An empty [`ChainRec`], from the pool or fresh.
@@ -21398,6 +21585,7 @@ impl Interpreter {
                 levels: Vec::new(),
                 parked: false,
                 pend_at: 0,
+                hold: None,
             })
         })
     }
@@ -21407,6 +21595,7 @@ impl Interpreter {
     fn chain_recycle(&mut self, mut rec: Box<ChainRec>) {
         debug_assert!(rec.levels.is_empty());
         rec.parked = false;
+        drop(rec.hold.take());
         if self.chain_pool.len() < 16 {
             self.chain_pool.push(rec);
         }
@@ -66448,6 +66637,10 @@ struct ChainRec {
     /// Where the levels' entries start on the pending list while the
     /// chain runs unflushed (see `Interpreter::collapse_flushed`).
     pend_at: usize,
+    /// The outermost level, while the chain runs for a `next()` call
+    /// (whose operands held it); otherwise its holder is the consumer's
+    /// stack, or (parked) whoever holds it.
+    hold: Option<Rc<PyGenerator>>,
 }
 
 // SAFETY: a record moves between threads with its generators, under the
diff --git a/tests/regrtest/test_yield_from_parked_chains.py b/tests/regrtest/test_yield_from_parked_chains.py
index ea33623f..e490e171 100644
--- a/tests/regrtest/test_yield_from_parked_chains.py
+++ b/tests/regrtest/test_yield_from_parked_chains.py
@@ -307,6 +307,60 @@ def tracer(frame, event, arg):
         )
         self.assertEqual(list(top), list(range(4, N)))
 
+    def test_temporary_chain_from_next(self):
+        log = []
+        for _ in range(20):
+            log.clear()
+            self.assertEqual(next(level(5, N, log)), 0)
+            # The chain's only owner was the call: released at once.
+            self.assertEqual(log, ["leaf", 0, 1, 2, 3, 4, 5])
+
+    def test_next_default_through_chain(self):
+        marker = object()
+        top = level(4, 3)
+        self.assertEqual([next(top, marker) for _ in range(5)], [0, 1, 2, marker, marker])
+
+    def test_for_loop_with_break(self):
+        top, gens = started(5, items=0)
+        seen = []
+        for x in top:
+            seen.append(x)
+            if x == 4:
+                break
+        for g in gens:
+            self.assertFalse(g.gi_running)
+            self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_SUSPENDED)
+        self.assertEqual(top.gi_frame.f_locals["depth"], 5)
+        for x in top:
+            seen.append(x)
+            if x == 9:
+                break
+        self.assertEqual(seen, list(range(10)))
+        self.assertEqual(next(top), 10)
+        self.assertEqual(list(top), list(range(11, N)))
+
+    def test_raise_through_for_loop(self):
+        def failing(n):
+            for i in range(n):
+                yield i
+            raise ValueError("deep")
+
+        def wrap(depth):
+            if depth == 0:
+                yield from failing(30)
+            else:
+                yield from wrap(depth - 1)
+
+        for _ in range(10):
+            out = []
+            try:
+                for x in wrap(4):
+                    out.append(x)
+            except ValueError as e:
+                names = [f.name for f in traceback.extract_tb(e.__traceback__)]
+            self.assertEqual(out, list(range(30)))
+            self.assertEqual(names, ["test_raise_through_for_loop"] + ["wrap"] * 5 + ["failing"])
+
     def test_generator_expression_consumer(self):
         top, gens = started(6)
         self.assertEqual(sum(x for x in top if x % 2), sum(i for i in range(3, N) if i % 2))

From 4291703d774adc04b09f1318bdd7878a0161a900 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 00:00:36 -0700
Subject: [PATCH 132/173] perf: step a simple delegate in place for yield from
 and await

A `yield from` or `await` whose delegate is a simple generator (one
`gen_fast` steps: a range loop, an `__await__` that yields once) resumed
that generator as an activation of its own at every item. Its fast step
now runs instead wherever the delegate is reached:

- the core loop's and the frame JIT's `SEND` of `None` step it in place,
  its yield taking the sent value's slot (a new `SEND_YIELDED` status
  lets compiled code go on at its own yield);
- a collapsed chain's resume steps its innermost generator when that one
  is simple (`chain_step`): the yield passes every level by and goes to
  the consumer directly, the chain staying parked, and a return lands in
  the innermost level's `SEND` exit, which then runs;
- a `FOR_ITER`, `next()` or `send(None)` of a generator whose own
  delegate is simple steps the delegate without touching the generator
  (`delegate_step`).

`gen.send(None)` and `coro.send(None)` calls now collapse a chain too.
`gen_fast_next` declines a body no fast step runs before anything else.

Measured (instructions retired per item, macOS):
- `for x in` a chain of plain `yield from` levels over a range
  generator: depth 2 1246 -> 1014, depth 16 1323 -> 977.
- `sum()` of such a chain: depth 2 1202 -> 805, depth 4 1409 -> 1016.
- `g.send(None)` on such a chain: depth 2 2096 -> 1348, depth 16
  2375 -> 1305.
- An `await` chain driven by `send(None)`, depth 4: 5944 -> 5069.
- Suite: `coroutines` 1.69x -> 1.65x CPython; `generators_tree` 1.53x
  -> 1.54x (callgrind: about 1% more per node in the checks).
---
 crates/weavepy-vm/src/frame_jit.rs           | 109 ++++-
 crates/weavepy-vm/src/gen_fast.rs            |  28 +-
 crates/weavepy-vm/src/lib.rs                 | 437 +++++++++++++++++--
 tests/regrtest/test_yield_from_fast_steps.py | 228 ++++++++++
 4 files changed, 747 insertions(+), 55 deletions(-)
 create mode 100644 tests/regrtest/test_yield_from_fast_steps.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index c223dc40..20e0ff18 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -1328,6 +1328,11 @@ unsafe extern "C" fn h_for_iter_rest(
                     sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
                     return FOR_SWITCHED;
                 }
+                // A collapsed chain's fast step delivered the value without
+                // a switch (see `Interpreter::chain_step`).
+                if std::ptr::eq(sw.cur, st.frame) {
+                    return if sw.pending.is_none() { 1 } else { FOR_SWITCHED };
+                }
                 // A compiled body runs from here; its yield (the value on
                 // top, past the instruction) or its return (the iterator
                 // gone, past the loop) continues this code natively.
@@ -5123,6 +5128,11 @@ unsafe extern "C" fn h_call(
             let interp = &mut *st.interp.cast_mut();
             if !interp.core_gen_resume(sw, pc, crate::InlineResume::SendCall) {
                 sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+            } else if std::ptr::eq(sw.cur, st.frame) && sw.pending.is_none() {
+                // A collapsed chain's fast step delivered the result (see
+                // `Interpreter::chain_step`).
+                st.len = (*st.frame).stack.len();
+                return 0;
             }
             return RELOAD;
         }
@@ -5169,6 +5179,11 @@ unsafe extern "C" fn h_call(
             let sw = &mut *st.sw;
             if !interp.core_gen_resume(sw, pc, crate::InlineResume::NextCall) {
                 sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+            } else if std::ptr::eq(sw.cur, st.frame) && sw.pending.is_none() {
+                // A collapsed chain's fast step delivered the result (see
+                // `Interpreter::chain_step`).
+                st.len = (*st.frame).stack.len();
+                return 0;
             }
             return RELOAD;
         }
@@ -5212,6 +5227,10 @@ unsafe extern "C" fn h_call(
             let sw = &mut *st.sw;
             if !interp.core_gen_resume(sw, pc, crate::InlineResume::NextDefault) {
                 sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
+            } else if std::ptr::eq(sw.cur, st.frame) && sw.pending.is_none() {
+                // As for `next(gen)` above.
+                st.len = (*st.frame).stack.len();
+                return 0;
             }
             return RELOAD;
         }
@@ -6196,13 +6215,18 @@ unsafe extern "C" fn h_await_iter(st: *mut State, at: *mut Object, kind: u64) ->
     }
 }
 
+/// [`h_send`]'s status for a delegate's fast step that yielded: the value
+/// in the sent one's place, for the code to go on at the `YIELD_VALUE`.
+const SEND_YIELDED: u32 = 2;
+
 /// `SEND` at `pc` of the running activation, its stack `len` deep: the
 /// generator or coroutine below the top resumed and switched to as the
 /// core loop's arm does (`RELOAD`), or, a compiled delegate run from here
-/// that returned into this `SEND`, its result in the sent value's place
-/// for the code to go on at the `END_SEND` (`0`); anything else (another
-/// delegate, or a generator's fast step, which never switches) declined
-/// untouched.
+/// (or a simple one's fast step) that returned into this `SEND`, its
+/// result in the sent value's place for the code to go on at the
+/// `END_SEND` (`0`), or a fast step's yield ([`SEND_YIELDED`]); anything
+/// else (another delegate, or a generator's fast step, which never
+/// switches) declined untouched.
 unsafe extern "C" fn h_send(st: *mut State, pc: u64, len: u64) -> u32 {
     // SAFETY: the code passes its live state and its stack depth (the
     // delegate and the sent value on top).
@@ -6218,16 +6242,31 @@ unsafe extern "C" fn h_send(st: *mut State, pc: u64, len: u64) -> u32 {
         {
             return 1;
         }
+        // A simple generator steps to its next yield in place, as the core
+        // loop's arm steps it (see `gen_fast`).
+        if let (Object::Generator(g), Object::None) = (&*st.stack.add(len - 2), &*st.stack.add(len - 1)) {
+            if crate::gen_fast::may_step(g) {
+                if let Some(r) = h_send_step(st, pc, len) {
+                    return r;
+                }
+            }
+        }
+        let interp = &mut *st.interp.cast_mut();
         let sw = &mut *st.sw;
         let frame = &mut *st.frame;
         frame.stack.set_len(len);
         frame.pc = pc as u32;
         *sw.last = st.last;
-        let interp = &mut *st.interp.cast_mut();
         if !interp.core_gen_resume(sw, pc as usize, crate::InlineResume::Send) {
             sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
             return RELOAD;
         }
+        // A collapsed chain's fast step delivered the yield without a
+        // switch (see `Interpreter::chain_step`).
+        if std::ptr::eq(sw.cur, st.frame) && sw.pending.is_none() {
+            st.len = len;
+            return SEND_YIELDED;
+        }
         if sw.pending.is_none() && !std::ptr::eq(sw.cur, st.frame) && run_switched_directly(st) {
             let frame = &*st.frame;
             let exit = pc as usize + 1 + frame.code.instructions[pc as usize].arg as usize;
@@ -6240,6 +6279,53 @@ unsafe extern "C" fn h_send(st: *mut State, pc: u64, len: u64) -> u32 {
     }
 }
 
+/// [`h_send`]'s fast step of the generator below the sent `None` (one a
+/// step may run): what it yields takes the sent value's slot (the code
+/// goes on at its yield, [`SEND_YIELDED`]), or its return value does as
+/// the `SEND` exits (`0`); `None` when the step declined, untouched.
+///
+/// # Safety
+///
+/// As [`h_send`]'s, the generator and `None` atop the stack.
+#[inline(never)]
+unsafe fn h_send_step(st: &mut State, pc: u64, len: usize) -> Option<u32> {
+    // SAFETY: the caller's contract.
+    unsafe {
+        let arg = (&*st.frame).code.instructions[pc as usize].arg;
+        if arg & crate::GEN_SEND != 0 {
+            return None;
+        }
+        let Object::Generator(g) = &*st.stack.add(len - 2) else {
+            return None;
+        };
+        let interp = &mut *st.interp.cast_mut();
+        let (v, exit) = match interp.gen_fast_next(g, st.snap_gen, 0, st.maybe_dead) {
+            crate::gen_fast::GenNext::Yielded(v) => (v, false),
+            crate::gen_fast::GenNext::Exhausted(v) => (v, true),
+            crate::gen_fast::GenNext::Declined | crate::gen_fast::GenNext::Partial => {
+                return None;
+            }
+        };
+        // The sent `None`'s slot (no drop glue owed).
+        st.stack.add(len - 1).write(v);
+        let to = pc as usize + 1 + if exit { arg as usize } else { 0 };
+        // Anything the step released may have queued a finalizer, run
+        // before the next instruction: the core loop takes over there,
+        // synced as for a switch.
+        if (*st.maybe_dead).get() {
+            let sw = &mut *st.sw;
+            let frame = &mut *st.frame;
+            frame.stack.set_len(len);
+            frame.pc = to as u32;
+            *sw.last = pc as usize;
+            sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Marked));
+            return Some(RELOAD);
+        }
+        st.len = len;
+        Some(if exit { 0 } else { SEND_YIELDED })
+    }
+}
+
 /// `YIELD_VALUE` at `pc` of the running activation, an inline generator
 /// resume (its stack `len` deep, the value on top), as the core loop's arm
 /// runs it: the generator parks and its consumer, the value delivered, is
@@ -8485,11 +8571,24 @@ impl<'a> Lower<'a> {
                 let switched = self.b.create_block();
                 let other = self.b.create_block();
                 let returned = self.b.create_block();
+                let rest = self.b.create_block();
+                let yielded = self.b.create_block();
                 let reload = self.b.ins().icmp_imm(IntCC::Equal, r, i64::from(RELOAD));
                 self.b.ins().brif(reload, switched, &[], other, &[]);
                 self.b.switch_to_block(switched);
                 self.b.ins().return_(&[r]);
                 self.b.switch_to_block(other);
+                // A delegate's fast step yielded: the value took the sent
+                // one's place, for this code's own `YIELD_VALUE`.
+                let is_yield = self
+                    .b
+                    .ins()
+                    .icmp_imm(IntCC::Equal, r, i64::from(SEND_YIELDED));
+                self.b.ins().brif(is_yield, yielded, &[], rest, &[]);
+                self.b.switch_to_block(yielded);
+                self.set_last(pc);
+                self.goto(next);
+                self.b.switch_to_block(rest);
                 self.b.ins().brif(r, out, &[], returned, &[]);
                 // The delegate returned into this `SEND`: its result took the
                 // sent value's place.
diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index e2b1a313..d541917c 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -144,7 +144,7 @@ const CODE_NATIVE: u8 = 3;
 /// (cached in the code's extension table). The prologue may hold others
 /// (`range(n)` built before the loop): the first resume's step stops at
 /// them, and the general loop runs them once.
-#[inline]
+#[inline(always)]
 fn code_ok(code: &CodeObject) -> u8 {
     use std::sync::atomic::Ordering;
     let Some(ext) = crate::code_vm_ext(code) else {
@@ -156,6 +156,19 @@ fn code_ok(code: &CodeObject) -> u8 {
     }
 }
 
+/// Whether a fast step may run `g`'s body at all (see [`code_ok`]).
+#[inline(always)]
+pub(crate) fn steps_body(g: &PyGenerator) -> bool {
+    matches!(&g.code, Object::Code(code) if code_ok(code) != CODE_NO)
+}
+
+/// Whether a fast step may run `g`'s body at all (see [`code_ok`]), and
+/// `g` isn't a collapsed chain's level (see `Interpreter::gen_fast_next`).
+#[inline(always)]
+pub(crate) fn may_step(g: &PyGenerator) -> bool {
+    g.chain.get() == 0 && steps_body(g)
+}
+
 /// [`code_ok`]'s first verdict for a code object, remembered in its
 /// extension table.
 #[cold]
@@ -1015,17 +1028,20 @@ impl Interpreter {
         depth: u8,
         dead: *const std::cell::Cell<bool>,
     ) -> GenNext {
+        // A level of a collapsed `yield from` chain is the inline resume's
+        // (which resumes a parked chain without settling it), and a body no
+        // fast step runs declines at once (its code's verdict is cached:
+        // see `code_ok`).
+        if g.chain.get() != 0 || matches!(&g.code, Object::Code(code) if code_ok(code) == CODE_NO)
+        {
+            return GenNext::Declined;
+        }
         if depth > MAX_DEPTH
             || crate::recursion::current_depth() + usize::from(depth) + 2
                 >= crate::recursion::recursion_limit()
         {
             return GenNext::Declined;
         }
-        // A level of a collapsed `yield from` chain is the inline resume's
-        // (which resumes a parked chain without settling it).
-        if g.chain.get() != 0 {
-            return GenNext::Declined;
-        }
         // Validate and take the frame under one exclusive view; no Python
         // runs while it's held.
         // SAFETY: nothing below reaches the cell again until `state`'s
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 3fbd2cc3..fc209bff 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -7755,7 +7755,8 @@ impl Interpreter {
                     Some(stop) => stop,
                     None => {
                         let calls_ok = self.inline_calls_ok();
-                        let mut sw = CoreSwitch::enter(roots, frame, shell, last, calls_ok);
+                        let mut sw =
+                            CoreSwitch::enter(roots, frame, shell, last, calls_ok, snap_gen);
                         let stop = self.leaf_burst(&mut sw, snap_gen);
                         if sw.switched() {
                             return FrameEv::Switched(stop);
@@ -12884,6 +12885,37 @@ impl Interpreter {
                             }
                             break Some(CoreExit::Stop(LeafStop::Step));
                         }
+                        // A simple generator steps to its next yield in
+                        // place, as a `FOR_ITER` over it does (see
+                        // `gen_fast`): what it yields takes the sent `None`'s
+                        // slot, or its return value does as the `SEND` exits.
+                        // SAFETY: `len >= 2` (checked above).
+                        if let (Object::Generator(g), Object::None, 0) = unsafe {
+                            (&*base.add(len - 2), &*base.add(len - 1), ins.arg & GEN_SEND)
+                        } {
+                            let (v, exit) = match gen_fast::may_step(g)
+                                .then(|| self.gen_fast_next(g, snap_gen, 0, pending_work))
+                            {
+                                None => (None, false),
+                                Some(gen_fast::GenNext::Yielded(v)) => (Some(v), false),
+                                Some(gen_fast::GenNext::Exhausted(v)) => (Some(v), true),
+                                Some(gen_fast::GenNext::Declined | gen_fast::GenNext::Partial) => {
+                                    (None, false)
+                                }
+                            };
+                            if let Some(v) = v {
+                                // SAFETY: the sent `None`'s slot (no drop
+                                // glue owed).
+                                unsafe { base.add(len - 1).write(v) };
+                                last = pc;
+                                pc += 1;
+                                if exit {
+                                    pc += ins.arg as usize;
+                                }
+                                after_release!();
+                                continue;
+                            }
+                        }
                         // SAFETY: `len <= cap`, every slot initialized.
                         unsafe { frame.stack.set_len(len) };
                         frame.pc = pc as u32;
@@ -20880,7 +20912,10 @@ impl Interpreter {
             },
             // A generator suspended in a `yield from` collapses its chain
             // as a `SEND` to it would (see `Self::core_iter_collapse`).
-            InlineResume::ForIter | InlineResume::NextCall | InlineResume::NextDefault => {
+            InlineResume::ForIter
+            | InlineResume::NextCall
+            | InlineResume::NextDefault
+            | InlineResume::SendCall => {
                 if self.core_iter_collapse(sw, pc, mode) == SendHop::Collapsed {
                     return true;
                 }
@@ -20888,11 +20923,6 @@ impl Interpreter {
                     return false;
                 }
             }
-            InlineResume::SendCall => {
-                if !self.core_gen_resume_one(sw, pc, mode) {
-                    return false;
-                }
-            }
         }
         // A `yield from` chain: a generator suspended in one resumes at
         // `RESUME; JUMP_BACKWARD_NO_INTERRUPT` back to its `SEND`, which
@@ -20927,12 +20957,7 @@ impl Interpreter {
         // SAFETY: see `CoreSwitch`: the running activation is synced, the
         // generator where `mode` puts it (checked by the caller).
         let frame = unsafe { &*sw.cur };
-        let n = frame.stack.len();
-        let at = match mode {
-            InlineResume::ForIter | InlineResume::NextCall => n.checked_sub(1),
-            _ => n.checked_sub(2),
-        };
-        let Some(Object::Generator(g)) = at.and_then(|at| frame.stack.get(at)) else {
+        let Some(g) = Self::resumed_gen(frame, mode) else {
             return SendHop::Declined;
         };
         if g.chain.get() != 0 {
@@ -20948,6 +20973,14 @@ impl Interpreter {
         let Some((next, next_send)) = Self::collapse_next(unsafe { &*gf }) else {
             return SendHop::Declined;
         };
+        // A simple delegate takes a fast step (see `Self::delegate_step`).
+        if gen_fast::may_step(next) {
+            let next: *const Rc<PyGenerator> = next;
+            // SAFETY: `next` lives on `g`'s stack, which the step leaves.
+            if let Some(hop) = self.delegate_step(sw, pc, mode, gf, unsafe { &*next }, next_send) {
+                return hop;
+            }
+        }
         // Its delegate heads a parked chain: the generator joins it.
         if let Some(addr) = Self::parked_head(next) {
             // SAFETY: `g` is suspended in a `yield from` of the chain's
@@ -20977,6 +21010,39 @@ impl Interpreter {
         }
     }
 
+    /// The generator a `FOR_ITER`, `next()` or `send(None)` call (`mode`)
+    /// atop `frame`'s stack resumes, if a collapsed chain may take it.
+    #[inline(always)]
+    fn resumed_gen(frame: &Frame, mode: InlineResume) -> Option<&Rc<PyGenerator>> {
+        let stack = &frame.stack;
+        let n = stack.len();
+        match mode {
+            InlineResume::ForIter | InlineResume::NextCall => match stack.last()? {
+                Object::Generator(g) => Some(g),
+                _ => None,
+            },
+            InlineResume::NextDefault => match stack.get(n.checked_sub(2)?)? {
+                Object::Generator(g) => Some(g),
+                _ => None,
+            },
+            // `gen.send(None)` / `coro.send(None)` (see `try_inline_gen`).
+            InlineResume::SendCall => {
+                if !matches!(stack.last()?, Object::None) {
+                    return None;
+                }
+                match (stack.get(n.checked_sub(3)?)?, &stack[n - 2]) {
+                    (Object::BoundMethod(bm), Object::Unbound) => match &bm.receiver {
+                        Object::Generator(g) | Object::Coroutine(g) => Some(g),
+                        _ => None,
+                    },
+                    (_, Object::Generator(g) | Object::Coroutine(g)) => Some(g),
+                    _ => None,
+                }
+            }
+            InlineResume::Send => None,
+        }
+    }
+
     /// One level of [`Self::core_gen_resume`]'s `yield from` chain: the
     /// running generator activation's `SEND` at `send_pc` to the suspended
     /// generator under its sent value, as [`Self::try_inline_gen`] resumes
@@ -21030,6 +21096,24 @@ impl Interpreter {
         if sent_none && !gf.sent_consumed && g.kind != crate::object::CoroutineKind::AsyncGenerator
         {
             if let Some((next, next_send)) = Self::collapse_next(gf) {
+                // A simple delegate of its own takes a fast step (see
+                // `Self::delegate_step`).
+                if gen_fast::may_step(next) {
+                    let (next, gf): (*const Rc<PyGenerator>, *mut Frame) =
+                        (next, std::ptr::from_ref(gf).cast_mut());
+                    // SAFETY: `next` lives on the delegate's stack, which the
+                    // step leaves; nothing else reaches its state meanwhile.
+                    if let Some(hop) = self.delegate_step(
+                        sw,
+                        send_pc,
+                        InlineResume::Send,
+                        gf,
+                        unsafe { &*next },
+                        next_send,
+                    ) {
+                        return hop;
+                    }
+                }
                 // Its delegate heads a parked chain: the delegate joins it.
                 if let Some(addr) = Self::parked_head(next) {
                     // SAFETY: `g` is suspended in a `yield from` of the
@@ -21290,6 +21374,18 @@ impl Interpreter {
                 inner_frame,
             )
         };
+        let (inner, inner_frame) = match self.chain_step(sw, send_pc, mode, &mut rec) {
+            ChainStep::No => (inner, inner_frame),
+            ChainStep::Inner(inner, inner_frame) => (inner, inner_frame),
+            ChainStep::Delivered => {
+                self.chain_repark(rec);
+                return true;
+            }
+            ChainStep::Resume => {
+                self.chain_recycle(rec);
+                return false;
+            }
+        };
         match self.collapse_commit(sw, send_pc, mode, rec, inner, inner_frame) {
             Ok(()) => true,
             Err(mut rec) => {
@@ -21303,6 +21399,207 @@ impl Interpreter {
         }
     }
 
+    /// The innermost generator of the collapsed chain `rec` (taken to run
+    /// for the consumer's `mode` at `pc`, see [`Self::collapse_commit`])
+    /// by fast step (see `gen_fast`) instead of as an activation: a yield
+    /// passes every level by, leaving each where it was, so it goes to
+    /// the consumer directly ([`ChainStep::Delivered`]); a return resumes
+    /// the innermost level in its `SEND`'s exit with the value
+    /// ([`ChainStep::Inner`], or [`ChainStep::Resume`] when that level is
+    /// the chain's only one, for the consumer to resume as usual).
+    #[inline(always)]
+    fn chain_step(
+        &mut self,
+        sw: &mut CoreSwitch,
+        pc: usize,
+        mode: InlineResume,
+        rec: &mut ChainRec,
+    ) -> ChainStep {
+        let last = rec.levels.last().expect("a collapsed chain");
+        // SAFETY: the innermost level is alive and delegating: its frame
+        // sits in its `SEND`, its delegate on top.
+        match (unsafe { &*last.act.frame }).stack.last() {
+            Some(Object::Generator(inner)) if gen_fast::may_step(inner) => {
+                let inner: *const Rc<PyGenerator> = inner;
+                self.chain_step_run(sw, pc, mode, rec, inner)
+            }
+            _ => ChainStep::No,
+        }
+    }
+
+    /// [`Self::chain_step`] for an innermost generator `inner` a fast step
+    /// may run.
+    #[inline(never)]
+    fn chain_step_run(
+        &mut self,
+        sw: &mut CoreSwitch,
+        pc: usize,
+        mode: InlineResume,
+        rec: &mut ChainRec,
+        inner: *const Rc<PyGenerator>,
+    ) -> ChainStep {
+        // (Its levels count against the recursion limit, as when resumed.)
+        // SAFETY: the running thread's own cell.
+        if unsafe { (*sw.depth_cell).get() } + rec.levels.len() + 2
+            >= crate::recursion::recursion_limit()
+        {
+            return ChainStep::No;
+        }
+        // SAFETY: held by the innermost level's stack, which nothing below
+        // changes before the step ends.
+        let inner = unsafe { &*inner };
+        match self.gen_fast_next(inner, sw.snap_gen, 0, sw.maybe_dead) {
+            gen_fast::GenNext::Yielded(v) => {
+                // The outermost level goes once the chain parks.
+                rec.hold = self.step_deliver(sw, pc, mode, v);
+                ChainStep::Delivered
+            }
+            gen_fast::GenNext::Exhausted(v) => {
+                // The innermost level's `SEND` exits with the return value,
+                // the level suspended there having taken it.
+                let level = rec.levels.pop().expect("a collapsed chain");
+                // SAFETY: alive and delegating (see `Delegate`); its frame
+                // just past its `SEND`, its delegate on top.
+                unsafe {
+                    level.suspend();
+                    let frame = &mut *level.act.frame;
+                    let jump = frame.code.instructions[level.send_pc].arg;
+                    frame.pc = level.send_pc as u32 + 1 + jump;
+                    frame.stack.push(v);
+                    frame.sent_consumed = true;
+                }
+                if rec.levels.is_empty() {
+                    ChainStep::Resume
+                } else {
+                    ChainStep::Inner(level.gen, level.act.frame)
+                }
+            }
+            gen_fast::GenNext::Declined | gen_fast::GenNext::Partial => ChainStep::No,
+        }
+    }
+
+    /// The value `v` a fast step of the generator the consumer's `mode`
+    /// at `pc` resumes (or of that generator's delegate) yielded, as the
+    /// consumer's result: its operands leave as the instruction's own
+    /// resume takes them, and the generator a `next()` or `send()` call
+    /// held is returned, for the caller to release once the chain is
+    /// settled.
+    fn step_deliver(
+        &mut self,
+        sw: &mut CoreSwitch,
+        pc: usize,
+        mode: InlineResume,
+        v: Object,
+    ) -> Option<Rc<PyGenerator>> {
+        // SAFETY: see `CoreSwitch`: the consumer is the running activation,
+        // synced, its operands as `mode` has them.
+        let frame = unsafe { &mut *sw.cur };
+        let mut held = None;
+        match mode {
+            InlineResume::Send => {
+                // The value takes the sent `None`'s place.
+                if let Some(top) = frame.stack.last_mut() {
+                    drop(std::mem::replace(top, v));
+                }
+            }
+            InlineResume::ForIter => frame.stack.push(v),
+            InlineResume::NextCall | InlineResume::NextDefault => {
+                if mode == InlineResume::NextDefault {
+                    drop(frame.stack.pop());
+                }
+                let (g, b, a) = (frame.stack.pop(), frame.stack.pop(), frame.stack.pop());
+                if let Some(Object::Generator(g)) = g {
+                    held = Some(g);
+                }
+                drop((a, b));
+                frame.stack.push(v);
+            }
+            InlineResume::SendCall => {
+                held = Self::send_call_pop(frame);
+                frame.stack.push(v);
+            }
+        }
+        frame.pc = pc as u32 + 1;
+        // SAFETY: as above.
+        unsafe { *sw.last = pc };
+        // Anything the step released may have queued a finalizer, run
+        // before the consumer's next instruction.
+        // SAFETY: the running thread's own flag.
+        if unsafe { (*sw.maybe_dead).get() } {
+            sw.pending = Some(CoreExit::Stop(LeafStop::Marked));
+        }
+        held
+    }
+
+    /// The resume (the consumer's `mode` at `pc`) of a generator (frame
+    /// `gf`) suspended in the `yield from` at `send_pc` of `inner`, a
+    /// simple generator, by `inner`'s fast step (see `gen_fast`): a yield
+    /// passes the generator by, leaving it where it was, and goes to the
+    /// consumer directly (`Collapsed`); a return lands in the generator's
+    /// `SEND` exit, which it takes as its sent value, for the consumer to
+    /// resume it as usual (`Declined`). `None` when the step declined.
+    #[inline(never)]
+    fn delegate_step(
+        &mut self,
+        sw: &mut CoreSwitch,
+        pc: usize,
+        mode: InlineResume,
+        gf: *mut Frame,
+        inner: &Rc<PyGenerator>,
+        send_pc: usize,
+    ) -> Option<SendHop> {
+        // (The generator counts against the recursion limit, as resumed.)
+        // SAFETY: the running thread's own cell.
+        if unsafe { (*sw.depth_cell).get() } + 3 >= crate::recursion::recursion_limit() {
+            return None;
+        }
+        match self.gen_fast_next(inner, sw.snap_gen, 0, sw.maybe_dead) {
+            gen_fast::GenNext::Yielded(v) => {
+                drop(self.step_deliver(sw, pc, mode, v));
+                Some(SendHop::Collapsed)
+            }
+            gen_fast::GenNext::Exhausted(v) => {
+                // SAFETY: the generator is suspended (checked by the
+                // caller), its frame at the `RESUME` after the yield.
+                unsafe {
+                    let frame = &mut *gf;
+                    let jump = frame.code.instructions[send_pc].arg;
+                    frame.pc = send_pc as u32 + 1 + jump;
+                    frame.stack.push(v);
+                    frame.sent_consumed = true;
+                }
+                Some(SendHop::Declined)
+            }
+            gen_fast::GenNext::Declined | gen_fast::GenNext::Partial => None,
+        }
+    }
+
+    /// A `send(None)` call's operands (see [`Self::resumed_gen`]) off
+    /// `frame`'s stack: the generator it resumes, which the caller keeps.
+    fn send_call_pop(frame: &mut Frame) -> Option<Rc<PyGenerator>> {
+        let (v, b, a) = (frame.stack.pop(), frame.stack.pop(), frame.stack.pop());
+        let g = match (&a, &b) {
+            (Some(Object::BoundMethod(bm)), _) => match &bm.receiver {
+                Object::Generator(g) | Object::Coroutine(g) => Some(g.clone()),
+                _ => None,
+            },
+            (_, Some(Object::Generator(g) | Object::Coroutine(g))) => Some(g.clone()),
+            _ => None,
+        };
+        drop((v, b, a));
+        g
+    }
+
+    /// Park `rec` again (see [`ChainRec`]) after its innermost generator's
+    /// yield went to the consumer directly ([`Self::chain_step`]).
+    fn chain_repark(&mut self, mut rec: Box<ChainRec>) {
+        rec.parked = true;
+        let hold = rec.hold.take();
+        let _ = Box::into_raw(rec);
+        // (Last: its release may settle the chain.)
+        drop(hold);
+    }
+
     /// Add `g` (frame `gf`, suspended in the `yield from` at `level_send`
     /// of `inner`, which `inner_frame` qualifies) to `rec` as its
     /// innermost level, and every generator below it that qualifies in
@@ -21415,6 +21712,9 @@ impl Interpreter {
         };
         debug_assert!(std::ptr::eq::<Frame>(&*boxed, inner_frame));
         boxed.gen_first_resume = false;
+        // (A step that stopped partway, or the innermost level's delegate's
+        // return delivered by one, already gave it its value.)
+        let sent_consumed = std::mem::take(&mut boxed.sent_consumed);
         // SAFETY: see `CoreSwitch`: the running activation is synced, its
         // stack as `mode` has it (checked by the caller).
         let frame = unsafe { &mut *sw.cur };
@@ -21447,10 +21747,15 @@ impl Interpreter {
                 drop((a, b));
                 GEN_NEXT_DEFAULT
             }
-            InlineResume::SendCall => unreachable!("a send call never collapses"),
+            InlineResume::SendCall => {
+                rec.hold = Self::send_call_pop(frame);
+                GEN_NEXT_CALL
+            }
         };
         // The resume's `None` goes in.
-        boxed.stack.push(Object::None);
+        if !sent_consumed {
+            boxed.stack.push(Object::None);
+        }
         frame.pc = pc as u32 + 1;
         let gen_frame: *mut Frame = &raw mut *boxed;
         let mut act = self.inline_slot();
@@ -21509,42 +21814,68 @@ impl Interpreter {
         // SAFETY: a parked record owns itself (see `ChainRec`); taken
         // here, put back below unless the chain runs.
         let mut rec = unsafe { Box::from_raw(addr as *mut ChainRec) };
-        let repark = |rec: Box<ChainRec>| {
+        rec.parked = false;
+        let repark = |mut rec: Box<ChainRec>| {
+            rec.parked = true;
             let _ = Box::into_raw(rec);
             SendHop::Declined
         };
         let last = rec.levels.last().expect("a collapsed chain");
         // SAFETY: the innermost level is alive and parked: its frame sits
-        // in its `SEND`, its delegate on top.
-        let (inner, inner_frame) = match unsafe { &*last.act.frame }.stack.last() {
-            Some(Object::Generator(g) | Object::Coroutine(g)) => match Self::collapse_frame(g) {
-                Some(frame) => (Rc::as_ptr(g), frame),
-                None => return repark(rec),
-            },
+        // in its `SEND`, its delegate on top (held there until it runs).
+        let inner: &Rc<PyGenerator> = match unsafe { &*last.act.frame }.stack.last() {
+            Some(Object::Generator(g) | Object::Coroutine(g)) => unsafe { &*std::ptr::from_ref(g) },
             _ => return repark(rec),
         };
-        // The innermost generator may since have started delegating
-        // (a `yield from` of a fresh generator): its own chain joins.
+        // A simple innermost generator takes a fast step first.
+        let stepped = if gen_fast::steps_body(inner) {
+            match self.chain_step_run(sw, pc, mode, &mut rec, inner) {
+                ChainStep::No => None,
+                ChainStep::Inner(inner, inner_frame) => Some((inner, inner_frame)),
+                ChainStep::Delivered => {
+                    self.chain_repark(rec);
+                    return SendHop::Collapsed;
+                }
+                ChainStep::Resume => {
+                    self.chain_recycle(rec);
+                    return SendHop::Declined;
+                }
+            }
+        } else {
+            None
+        };
         let base = rec.levels.len();
-        // SAFETY: as `core_send_collapse` (the innermost generator was
-        // just qualified, and with it the generator it delegates to).
-        let (inner, inner_frame) = match Self::collapse_next(unsafe { &*inner_frame }) {
-            Some((next, next_send)) => match Self::collapse_frame(next) {
-                Some(next_frame) => unsafe {
-                    Self::collapse_walk(
-                        &mut rec,
-                        inner,
-                        inner_frame,
-                        Rc::as_ptr(next),
-                        next_send,
-                        next_frame,
-                    )
-                },
-                None => (inner, inner_frame),
-            },
-            None => (inner, inner_frame),
+        let (inner, inner_frame) = match stepped {
+            // The innermost level took its delegate's return value.
+            Some(stepped) => stepped,
+            None => {
+                let (inner, inner_frame) = match Self::collapse_frame(inner) {
+                    Some(frame) => (Rc::as_ptr(inner), frame),
+                    None => return repark(rec),
+                };
+                // The innermost generator may since have started delegating
+                // (a `yield from` of a fresh generator): its own chain joins.
+                // SAFETY: as `core_send_collapse` (the innermost generator
+                // was just qualified, and with it the generator it
+                // delegates to).
+                match Self::collapse_next(unsafe { &*inner_frame }) {
+                    Some((next, next_send)) => match Self::collapse_frame(next) {
+                        Some(next_frame) => unsafe {
+                            Self::collapse_walk(
+                                &mut rec,
+                                inner,
+                                inner_frame,
+                                Rc::as_ptr(next),
+                                next_send,
+                                next_frame,
+                            )
+                        },
+                        None => (inner, inner_frame),
+                    },
+                    None => (inner, inner_frame),
+                }
+            }
         };
-        rec.parked = false;
         match self.collapse_commit(sw, pc, mode, rec, inner, inner_frame) {
             Ok(()) => SendHop::Collapsed,
             Err(mut rec) => {
@@ -21552,7 +21883,6 @@ impl Interpreter {
                     // SAFETY: delegating since the walk.
                     unsafe { level.suspend() };
                 }
-                rec.parked = true;
                 repark(rec)
             }
         }
@@ -66837,7 +67167,7 @@ const GEN_NEXT_CALL: u32 = u32::MAX;
 /// Set in [`InlineAct::exhaust_arg`] (with the `SEND`'s jump below it) for
 /// a sub-generator resumed by `yield from` or `await`: exhaustion leaves
 /// its return value on the delegator's stack and takes the jump.
-const GEN_SEND: u32 = 1 << 31;
+pub(crate) const GEN_SEND: u32 = 1 << 31;
 
 /// [`InlineAct::exhaust_arg`] of a generator resumed by `next(gen,
 /// default)`: the default waits atop the caller's stack, replaced by a
@@ -66845,6 +67175,21 @@ const GEN_SEND: u32 = 1 << 31;
 /// finishes. (Checked before [`GEN_SEND`], whose bit it has.)
 const GEN_NEXT_DEFAULT: u32 = u32::MAX - 1;
 
+/// How [`Interpreter::chain_step`] went.
+enum ChainStep {
+    /// Nothing happened.
+    No,
+    /// The innermost generator yielded, its value delivered to the
+    /// consumer: the chain parks again.
+    Delivered,
+    /// The innermost generator returned into the innermost level, now the
+    /// chain's innermost generator (its frame), to run.
+    Inner(*const PyGenerator, *mut Frame),
+    /// The innermost generator returned into the chain's only level, out
+    /// of the chain: the consumer resumes it as usual.
+    Resume,
+}
+
 /// How [`Interpreter::core_send_hop`] went.
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum SendHop {
@@ -66973,6 +67318,8 @@ struct CoreSwitch {
     /// or bump the loop generation when they change (the eval-frame
     /// recorder), which every direct call compares first.
     calls_ok: bool,
+    /// The loop generation the burst runs under (see `quiet_run`).
+    snap_gen: u64,
 }
 
 impl CoreSwitch {
@@ -66983,6 +67330,7 @@ impl CoreSwitch {
         shell: &mut QuietShell<'_>,
         last: &mut usize,
         calls_ok: bool,
+        snap_gen: u64,
     ) -> Self {
         let frame: *mut Frame = frame;
         let shell: *mut QuietShell<'static> = std::ptr::from_mut(shell).cast();
@@ -67008,6 +67356,7 @@ impl CoreSwitch {
             depth_cell: crate::recursion::depth_cell(),
             obs_gen: crate::trace::observer_gen(),
             calls_ok,
+            snap_gen,
         }
     }
 
diff --git a/tests/regrtest/test_yield_from_fast_steps.py b/tests/regrtest/test_yield_from_fast_steps.py
new file mode 100644
index 00000000..602de0b0
--- /dev/null
+++ b/tests/regrtest/test_yield_from_fast_steps.py
@@ -0,0 +1,228 @@
+"""`yield from` and `await` of simple generators stepped in place.
+
+A `SEND` to a simple generator, or the resume of a collapsed chain whose
+innermost generator is a simple one, may run that generator's body to its
+next yield without an activation: the value passes every level by, and a
+return lands in the innermost level's `SEND`. Values, return values,
+`StopIteration`, the levels' state and frames, and every consumer
+(`for`, `next()`, `send(None)`, `sum()`, coroutine drivers, asyncio) must
+see exactly what resuming each level in turn shows.
+"""
+
+import asyncio
+import inspect
+import types
+import unittest
+
+N = 40
+
+
+def counting(n):
+    for i in range(n):
+        yield i
+    return n
+
+
+def relay(depth, n, log=None):
+    if depth == 0:
+        got = yield from counting(n)
+    else:
+        got = yield from relay(depth - 1, n, log)
+    if log is not None:
+        log.append((depth, got))
+    return got + 1
+
+
+def twice(n):
+    a = yield from counting(n)
+    b = yield from counting(n)
+    return a + b
+
+
+class YieldFromFastStepsTest(unittest.TestCase):
+    def test_values_and_returns(self):
+        for depth in (0, 1, 3, 12):
+            log = []
+            g = relay(depth, N, log)
+            self.assertEqual(list(g), list(range(N)))
+            self.assertEqual(log, [(d, N + d) for d in range(depth + 1)])
+
+    def test_return_value_reaches_consumer(self):
+        for depth in (0, 2, 6):
+            g = relay(depth, 3)
+            self.assertEqual([next(g), next(g), next(g)], [0, 1, 2])
+            with self.assertRaises(StopIteration) as cm:
+                next(g)
+            self.assertEqual(cm.exception.value, 3 + depth + 1)
+
+    def test_sequential_delegations(self):
+        def outer():
+            total = yield from twice(5)
+            yield ("total", total)
+
+        for _ in range(50):
+            self.assertEqual(list(outer()), list(range(5)) * 2 + [("total", 10)])
+
+    def test_consumers(self):
+        for depth in (1, 4):
+            self.assertEqual(sum(relay(depth, N)), sum(range(N)))
+            g = relay(depth, N)
+            self.assertEqual([next(g, None) for _ in range(N + 2)], list(range(N)) + [None, None])
+            g = relay(depth, N)
+            out = []
+            try:
+                while True:
+                    out.append(g.send(None))
+            except StopIteration as e:
+                self.assertEqual(e.value, N + depth + 1)
+            self.assertEqual(out, list(range(N)))
+
+    def test_send_values_mixed(self):
+        def echo():
+            got = None
+            for _ in range(N):
+                got = yield got
+            return "done"
+
+        def wrap(depth):
+            if depth == 0:
+                return (yield from echo())
+            return (yield from wrap(depth - 1))
+
+        g = wrap(4)
+        self.assertIsNone(next(g))
+        seen = []
+        for i in range(N - 1):
+            seen.append(g.send(i if i % 3 else None))
+        self.assertEqual(seen, [i if i % 3 else None for i in range(N - 1)])
+        with self.assertRaises(StopIteration) as cm:
+            g.send(None)
+        self.assertEqual(cm.exception.value, "done")
+
+    def test_levels_between_items(self):
+        gens = []
+
+        def level(depth):
+            if depth == 0:
+                inner = counting(N)
+            else:
+                inner = level(depth - 1)
+            gens.append(inner)
+            yield from inner
+
+        top = level(4)
+        for _ in range(5):
+            next(top)
+        self.assertEqual(len(gens), 5)
+        for g in [top] + gens[:-1]:
+            self.assertTrue(g.gi_suspended)
+            self.assertIsNotNone(g.gi_yieldfrom)
+            self.assertEqual(g.gi_frame.f_code.co_name, "level")
+        inner = gens[-1]
+        self.assertEqual(inner.gi_frame.f_locals["i"], 4)
+        self.assertEqual(next(top), 5)
+        self.assertEqual(next(inner), 6)
+        self.assertEqual(list(top), list(range(7, N)))
+
+    def test_close_and_throw_reach_the_simple_generator(self):
+        log = []
+
+        def guarded(n):
+            try:
+                for i in range(n):
+                    yield i
+            finally:
+                log.append("closed")
+
+        def wrap(depth):
+            if depth == 0:
+                yield from guarded(N)
+            else:
+                yield from wrap(depth - 1)
+
+        g = wrap(3)
+        self.assertEqual([next(g) for _ in range(4)], [0, 1, 2, 3])
+        g.close()
+        self.assertEqual(log, ["closed"])
+        g = wrap(3)
+        next(g)
+        with self.assertRaises(KeyError):
+            g.throw(KeyError)
+        self.assertEqual(log, ["closed", "closed"])
+
+    def test_await_simple_awaitables(self):
+        class Tick:
+            def __await__(self):
+                got = yield "tick"
+                return got
+
+        async def ticking(n):
+            total = 0
+            for i in range(n):
+                total += await Tick() or 0
+            return total
+
+        async def wrap(depth, n):
+            if depth == 0:
+                return await ticking(n)
+            return await wrap(depth - 1, n)
+
+        for depth in (0, 3):
+            coro = wrap(depth, N)
+            sent = []
+            value = coro.send(None)
+            try:
+                k = 0
+                while True:
+                    sent.append(value)
+                    k += 1
+                    value = coro.send(None if k % 2 else k)
+            except StopIteration as e:
+                result = e.value
+            self.assertEqual(sent, ["tick"] * N)
+            self.assertEqual(result, sum(k for k in range(1, N + 1) if k % 2 == 0))
+
+    def test_types_coroutine_chain(self):
+        @types.coroutine
+        def step():
+            yield
+
+        async def inner(n):
+            for _ in range(n):
+                await step()
+            return n
+
+        async def outer(n):
+            return await inner(n) + 1
+
+        coro = outer(N)
+        count = 0
+        try:
+            while True:
+                self.assertIsNone(coro.send(None))
+                count += 1
+                if count == N // 2:
+                    self.assertEqual(inspect.getcoroutinestate(coro), inspect.CORO_SUSPENDED)
+                    self.assertIsNotNone(coro.cr_await)
+        except StopIteration as e:
+            self.assertEqual(e.value, N + 1)
+        self.assertEqual(count, N)
+
+    def test_asyncio_sleep_zero(self):
+        async def worker(k, log):
+            for i in range(10):
+                await asyncio.sleep(0)
+                log.append((k, i))
+
+        async def main():
+            log = []
+            await asyncio.gather(*(worker(k, log) for k in range(3)))
+            return log
+
+        log = asyncio.run(main())
+        self.assertEqual(sorted(log), [(k, i) for k in range(3) for i in range(10)])
+        self.assertEqual([i for k, i in log if k == 0], list(range(10)))
+
+
+if __name__ == "__main__":
+    unittest.main()

From 54c9a3db5b72ab2f53b618c2fd2013c6b9ff2b47 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 00:20:26 -0700
Subject: [PATCH 133/173] perf: park a fast-stepped generator without
 re-checking its state

`gen_fast_next` called out to `gen_fast_frame_ok` and, after the step, to
`park_suspended_boxed`, which re-validates the generator's state cell and
stamps a frame object it can't have. The frame check is now inlined (its
code verdict shared with the step's early decline), and a step's own
generator parks with a plain store unless it still has to be handed to
the collector.

Measured (instructions retired per item, macOS):
- `tuple(x for x in data)`: 987 -> 944.
- `for x in gen(n)`: 713 -> 663.
- `t += next(g)` in a compiled loop: 1014 -> 961.
---
 crates/weavepy-vm/src/gen_fast.rs | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index d541917c..23d9052d 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -381,7 +381,7 @@ impl Interpreter {
     /// fast-step body the general machinery holds no extra state for (no
     /// Python-visible frame, no saved exception state, no parked native
     /// activation).
-    #[inline]
+    #[inline(always)]
     pub(crate) fn gen_fast_frame_ok(frame: &Frame) -> bool {
         frame.py_frame.is_none()
             && !frame.has_saved_exc_info()
@@ -1077,7 +1077,7 @@ impl Interpreter {
         #[cfg(feature = "jit")]
         let handed = match self.gen_native_yield(frame, snap_gen, depth, dead) {
             Ok(v) => {
-                Self::park_suspended_boxed(g, boxed);
+                Self::park_stepped(g, boxed);
                 return GenNext::Yielded(v);
             }
             Err(pc) => pc,
@@ -1119,7 +1119,26 @@ impl Interpreter {
                 GenNext::Partial
             }
         };
-        Self::park_suspended_boxed(g, boxed);
+        Self::park_stepped(g, boxed);
         out
     }
+
+    /// [`Self::park_suspended_boxed`] for a generator a fast step ran:
+    /// its state is still the step's `Running` (no code ran that could
+    /// reach it), and it has no frame object to stamp.
+    #[inline(always)]
+    fn park_stepped(g: &Rc<PyGenerator>, boxed: Box<Frame>) {
+        debug_assert!(boxed.py_frame.is_none());
+        if g.track_deferred.get() {
+            Self::park_suspended_boxed(g, boxed);
+            return;
+        }
+        // SAFETY: as above: nothing borrows the state, and `Running` owns
+        // nothing (no drop glue for the displaced state).
+        unsafe {
+            let st = g.state_raw().as_ptr();
+            debug_assert!(matches!(*st, GeneratorState::Running));
+            std::ptr::write(st, GeneratorState::Suspended(boxed));
+        }
+    }
 }

From ee5121b34d4fe2ad6e164fb9aeb3a782c7a43d20 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 01:04:07 -0700
Subject: [PATCH 134/173] perf: keep native fast steps to next()-like consumers

The native consumers' fast step (an earlier commit) sat in
`generator_send_fold`, so a native `gen.send(None)` stepped a simple body
in place too, and a body tier 2 parks natively never reached its parked
activation: the VM unit test
`jit_worker_exit_preserves_shared_code_and_parked_generators` failed
(its generators' parked activations no longer materialized), and a fast
step could meet a body a core-loop step left partway with an empty
stack, which tripped a debug assertion.

The step now runs for `next()` and the native iterators only
(`generator_next_raw`: builtin `next` with or without a default outside
the core loop, `iter_next`), and a partial body's resume no longer
asserts a non-empty stack. VM unit tests: 428 passed (11 failed before).

Measured (instructions retired per item, macOS): unchanged within noise
(`for i, x in enumerate(gen(n))` 2248 -> 2216, `next()` 961 -> 967).
---
 crates/weavepy-vm/src/gen_fast.rs |  3 +-
 crates/weavepy-vm/src/lib.rs      | 60 +++++++++++++++++++------------
 2 files changed, 40 insertions(+), 23 deletions(-)

diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index 23d9052d..8756db3f 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -1072,7 +1072,8 @@ impl Interpreter {
         } else {
             crate::push_fast(&mut frame.stack, Object::None);
         }
-        debug_assert!(!frame.stack.is_empty());
+        // (A partial body may have stopped with nothing on its stack.)
+        debug_assert!(partial || !frame.stack.is_empty());
         // A compiled body's resume usually runs natively to its next yield.
         #[cfg(feature = "jit")]
         let handed = match self.gen_native_yield(frame, snap_gen, depth, dead) {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index fc209bff..26aaf972 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -39521,7 +39521,7 @@ impl Interpreter {
                     None => Ok(default.expect("checked above")),
                 }
             }
-            Object::Generator(g) => self.generator_send(g, Object::None),
+            Object::Generator(g) => self.generator_next_raw(g, false),
             Object::Coroutine(_) => match self.gen_method_send(it, Object::None) {
                 Ok(v) => Ok(v),
                 Err(RuntimeError::PyException(exc)) if exc.type_name() == "StopIteration" => {
@@ -42869,7 +42869,7 @@ impl Interpreter {
         match iter {
             Object::Iter(it) => it.borrow_mut().next_value_checked(),
             Object::LazyIter(l) => self.lazy_iter_next(l, globals),
-            Object::Generator(g) => match self.generator_send(g, Object::None) {
+            Object::Generator(g) => match self.generator_next_raw(g, false) {
                 Ok(v) => Ok(Some(v)),
                 Err(RuntimeError::PyException(exc)) if exc.type_name() == "StopIteration" => {
                     Ok(None)
@@ -44944,10 +44944,33 @@ impl Interpreter {
     /// on the frame or its cached shell), it suspended outside any
     /// handler (no detached exception state to restore), and its code
     /// is the interpreter's — else `None` and the general path runs.
-    /// [`Self::generator_send_fold`]'s resume of `gen` with `None` by fast
-    /// step (see `gen_fast`), on the lean resume's terms: `None` when the
-    /// generator or its body declines, with nothing run (or the body
-    /// stopped partway, parked for the lean resume to go on with).
+    /// `next(gen)` from native code (`next()` outside the core loop, a
+    /// native iterator's step over a generator): a simple body steps to
+    /// its next yield in place, as a `FOR_ITER` over it does (see
+    /// `gen_fast`), and anything else resumes through
+    /// [`Self::generator_send`]. With `raw`, as [`Self::generator_resume`]
+    /// reports a return. (`gen.send(None)` keeps its activation: a body
+    /// tier 2 parks natively resumes there.)
+    fn generator_next_raw(
+        &mut self,
+        gen: &Rc<PyGenerator>,
+        raw: bool,
+    ) -> Result<Object, RuntimeError> {
+        if gen.kind == crate::object::CoroutineKind::Generator {
+            if let Some(r) = self.generator_next_fast(gen, raw) {
+                return r;
+            }
+        }
+        if raw {
+            GEN_RAW_RETURN.with(|r| r.set(true));
+        }
+        self.generator_send_fold(gen, Object::None, None)
+    }
+
+    /// [`Self::generator_next_raw`]'s fast step, on the lean resume's
+    /// terms: `None` when the generator or its body declines, with nothing
+    /// run (or the body stopped partway, parked for the general resume to
+    /// go on with).
     #[inline]
     fn generator_next_fast(
         &mut self,
@@ -45268,10 +45291,15 @@ impl Interpreter {
         fire: bool,
     ) -> Result<Option<Object>, RuntimeError> {
         let raw = !crate::trace::any_observers_active();
-        if raw {
-            GEN_RAW_RETURN.with(|r| r.set(true));
-        }
-        match self.generator_send_fold(g, Object::None, fold) {
+        let result = if fold.is_none() {
+            self.generator_next_raw(g, raw)
+        } else {
+            if raw {
+                GEN_RAW_RETURN.with(|r| r.set(true));
+            }
+            self.generator_send_fold(g, Object::None, fold)
+        };
+        match result {
             Ok(Object::Unbound) if raw => {
                 drop(GEN_RETURN_VALUE.with(|v| v.borrow_mut().take()));
                 Ok(None)
@@ -45311,18 +45339,6 @@ impl Interpreter {
         // Consumed here, so a generator this one resumes in turn raises
         // `StopIteration` as usual.
         let raw_return = GEN_RAW_RETURN.with(|r| r.replace(false));
-        // A simple body's resume steps to its next yield in place, as a
-        // `FOR_ITER` over it does (see `gen_fast`): no activation at all.
-        // (A draining consumer's fold takes the lean resume, which folds
-        // every yield it can in one step.)
-        if fold.is_none()
-            && matches!(sent, Object::None)
-            && gen.kind == crate::object::CoroutineKind::Generator
-        {
-            if let Some(r) = self.generator_next_fast(gen, raw_return) {
-                return r;
-            }
-        }
         if let Some(r) = self.generator_send_lean(gen, &sent, fold, raw_return) {
             return r;
         }

From 05fa45fc2e83999c98495d100ead8349051dad1d Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 01:55:47 -0700
Subject: [PATCH 135/173] perf: skip doomed first fast steps and finish lean
 fast runs in place

A fresh generator's first resume tried a fast step even when its
prologue (everything before the first `yield`) holds an instruction the
step can't run, so the step stopped partway and the general resume took
over: `iter(obj)` of a class whose `__iter__` reads an attribute before
its yield paid for both. `code_ok`'s scan now also records a verdict for
the prologue, and a first resume whose prologue a step can't finish
(or needs native code not yet compiled) goes straight to the general
resume.

The lean resume's fast run (`gen_fast_run`, a native drain such as
`sum(genexpr)`) now runs a body's `return` too, so an exhausted
generator finishes there instead of re-entering the quiet loop for its
last two instructions.

Measured (instructions retired per item, macOS):
- `for x in node` (an `__iter__` generator per loop): 4570 -> 4134.
- `sum(x for x in pair)` per call: 8033 -> 7379.
- `for x in outer(node)` (`yield from` over it): 7873 -> 7588.
- Suite: unchanged within noise (geomean of 8 generator-heavy
  workloads 1.001x of the previous commit).
---
 crates/weavepy-vm/src/gen_fast.rs | 110 ++++++++++++++++++++++--------
 crates/weavepy-vm/src/lib.rs      |   6 +-
 2 files changed, 85 insertions(+), 31 deletions(-)

diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index 8756db3f..f9431979 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -137,6 +137,10 @@ const CODE_NO: u8 = 1;
 const CODE_OK: u8 = 2;
 /// Fast steps once the body is compiled (see [`op_native`]).
 const CODE_NATIVE: u8 = 3;
+/// The verdict's bits in the table's byte; the prologue's verdict (see
+/// [`prologue`]) sits above them.
+const CODE_MASK: u8 = 0x0f;
+const PROLOGUE_SHIFT: u32 = 4;
 
 /// Whether `code` is a plain generator whose steady state, everything a
 /// resume can reach from a `yield`, is instructions a fast step runs
@@ -151,9 +155,27 @@ fn code_ok(code: &CodeObject) -> u8 {
         return CODE_NO;
     };
     match ext.gen_fast.load(Ordering::Relaxed) {
+        0 => code_ok_scan(code, ext) & CODE_MASK,
+        v => v & CODE_MASK,
+    }
+}
+
+/// [`code_ok`]'s verdict for a body's first resume, everything it can
+/// reach before its first `yield`: [`CODE_OK`] when a fast step runs all
+/// of it, [`CODE_NATIVE`] when only its native form does, [`CODE_NO`]
+/// when the step would stop partway (the general loop runs a prologue
+/// with a call, say, once).
+#[inline(always)]
+fn prologue(code: &CodeObject) -> u8 {
+    use std::sync::atomic::Ordering;
+    let Some(ext) = crate::code_vm_ext(code) else {
+        return CODE_NO;
+    };
+    let v = match ext.gen_fast.load(Ordering::Relaxed) {
         0 => code_ok_scan(code, ext),
         v => v,
-    }
+    };
+    v >> PROLOGUE_SHIFT
 }
 
 /// Whether a fast step may run `g`'s body at all (see [`code_ok`]).
@@ -180,30 +202,43 @@ fn code_ok_scan(code: &CodeObject, ext: &crate::CodeConstObjects) -> u8 {
         && !code.is_async_generator
         && code.cellvars.is_empty()
         && in_bounds(code, ext.objects.len());
-    let v = if !shape {
-        CODE_NO
-    } else if code.freevars.is_empty() && steady_state_supported(code, false) {
-        CODE_OK
-    } else if cfg!(feature = "jit") && steady_state_supported(code, true) {
-        CODE_NATIVE
-    } else {
-        CODE_NO
+    let verdict = |starts: &[usize], stop_at_yield: bool| {
+        if !shape {
+            CODE_NO
+        } else if code.freevars.is_empty() && reach_supported(code, starts, false, stop_at_yield) {
+            CODE_OK
+        } else if cfg!(feature = "jit") && reach_supported(code, starts, true, stop_at_yield) {
+            CODE_NATIVE
+        } else {
+            CODE_NO
+        }
+    };
+    let instrs = &code.instructions;
+    let resumes: Vec<usize> = (0..instrs.len())
+        .filter(|&p| instrs[p].op == OpCode::YieldValue)
+        .map(|p| p + 1)
+        .collect();
+    let v = verdict(&resumes, false);
+    // A fresh generator starts past its `RETURN_GENERATOR`.
+    let start = instrs.iter().position(|i| i.op == OpCode::ReturnGenerator);
+    let first = match (v, start) {
+        (CODE_NO, _) | (_, None) => CODE_NO,
+        (_, Some(at)) => verdict(&[at + 1], true),
     };
+    let v = v | (first << PROLOGUE_SHIFT);
     ext.gen_fast.store(v, Ordering::Relaxed);
     v
 }
 
-/// Whether every instruction a resume reaches from one of `code`'s yields
-/// is one a fast step runs (jump targets are in range: see [`in_bounds`]),
-/// or with `native`, one its native form runs (see [`op_native`]).
-fn steady_state_supported(code: &CodeObject, native: bool) -> bool {
+/// Whether every instruction a run from one of `starts` reaches (with
+/// `stop_at_yield`, before a `yield`) is one a fast step runs (jump
+/// targets are in range: see [`in_bounds`]), or with `native`, one its
+/// native form runs (see [`op_native`]).
+fn reach_supported(code: &CodeObject, starts: &[usize], native: bool, stop_at_yield: bool) -> bool {
     let instrs = &code.instructions;
     let n = instrs.len();
     let mut seen = vec![false; n];
-    let mut work: Vec<usize> = (0..n)
-        .filter(|&p| instrs[p].op == OpCode::YieldValue)
-        .map(|p| p + 1)
-        .collect();
+    let mut work = starts.to_vec();
     while let Some(p) = work.pop() {
         if p >= n || std::mem::replace(&mut seen[p], true) {
             continue;
@@ -215,6 +250,7 @@ fn steady_state_supported(code: &CodeObject, native: bool) -> bool {
         let arg = ins.arg as usize;
         match ins.op {
             OpCode::ReturnValue | OpCode::Reraise => {}
+            OpCode::YieldValue if stop_at_yield => {}
             OpCode::JumpForward => work.push(p + 1 + arg),
             OpCode::JumpBackward => work.push((p + 1).saturating_sub(arg)),
             OpCode::PopJumpIfFalse
@@ -992,29 +1028,45 @@ impl Interpreter {
 
     /// A resume's fast steps (its sent value already pushed): the value
     /// the body yields next, with a draining consumer's `fold` taking the
-    /// yields it can along the way, or `None` for the general loop to
-    /// continue from wherever the steps stopped.
+    /// yields it can along the way, or the value it returns, or
+    /// [`GenStep::Bail`] for the general loop to continue from wherever
+    /// the steps stopped.
     pub(crate) fn gen_fast_run(
         &mut self,
         frame: &mut Frame,
         snap_gen: u64,
         fold: Option<FoldSink>,
-    ) -> Option<Object> {
-        if !Self::gen_fast_frame_ok(frame) {
-            return None;
+    ) -> GenStep {
+        if !Self::gen_fast_frame_ok(frame)
+            || (frame.gen_first_resume && !Self::gen_fast_prologue_ok(frame))
+        {
+            return GenStep::Bail;
         }
-        match self.gen_fast_step(
+        self.gen_fast_step(
             frame,
             snap_gen,
             0,
             fold,
             crate::gc_trace::maybe_dead_flag(),
-            false,
+            true,
             usize::MAX,
-        ) {
-            GenStep::Yielded(v) => Some(v),
-            GenStep::Returned(_) => unreachable!("not a finishing step"),
-            GenStep::Bail => None,
+        )
+    }
+
+    /// Whether a fresh generator's first resume (on `frame`) can take a
+    /// fast step to its first yield (see [`prologue`]): one that would
+    /// stop partway leaves the whole resume to the general loop.
+    #[inline(always)]
+    fn gen_fast_prologue_ok(frame: &Frame) -> bool {
+        match prologue(&frame.code) {
+            CODE_OK => true,
+            #[cfg(feature = "jit")]
+            CODE_NATIVE => crate::code_vm_ext(&frame.code).is_some_and(|ext| {
+                // SAFETY: a length read with nothing running.
+                let n = unsafe { (*frame.locals.as_ptr()).len() };
+                ext.frame_jit.get(n).is_some()
+            }),
+            _ => false,
         }
     }
 
@@ -1053,7 +1105,7 @@ impl Interpreter {
             return GenNext::Declined;
         };
         let first_resume = matches!(*state, GeneratorState::Created(_));
-        if !Self::gen_fast_frame_ok(boxed) {
+        if !Self::gen_fast_frame_ok(boxed) || (first_resume && !Self::gen_fast_prologue_ok(boxed)) {
             return GenNext::Declined;
         }
         // A body an earlier fast step left partway (at the eval breaker,
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 26aaf972..d39e730f 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -45073,7 +45073,7 @@ impl Interpreter {
             // single send would then finish the whole generator on the
             // slow path. Service the breaker here (as the quiet loop's
             // checkpoint does) and keep stepping.
-            while fast.is_none()
+            while matches!(fast, gen_fast::GenStep::Bail)
                 && self.gil_countdown <= 1
                 && crate::hot_gates::loop_gen() == snap_gen
                 && !frame.sent_consumed
@@ -45085,8 +45085,10 @@ impl Interpreter {
                 }
                 fast = self.gen_fast_run(frame, snap_gen, fold);
             }
-            if let Some(v) = fast {
+            if let gen_fast::GenStep::Yielded(v) = fast {
                 Ok(FrameOutcome::Yielded(v))
+            } else if let gen_fast::GenStep::Returned(v) = fast {
+                Ok(FrameOutcome::Returned(v))
             } else {
                 // The draining consumer's sink, folded at this frame's yields.
                 let prev_fold = std::mem::replace(

From e78f5e20e4e26c1a3a04cdcd4a9d574759022ddc Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:15:57 -0700
Subject: [PATCH 136/173] perf: look before collapsing a chain from FOR_ITER
 and next()

Every `FOR_ITER`, `next()` or `send(None)` of a generator the fast step
declined called `core_iter_collapse`, which qualified the generator as a
chain level before finding it wasn't suspended in a `yield from` at all.
An inline first look (a chain's level, or a suspended frame at a
`yield from`'s resume) now skips the call for every other generator.

Measured (instructions retired per item, macOS): `for x in` a recursive
generator that isn't delegating (one level) 1247 -> 1177 (1188 before
the chain work); the chain probes unchanged within noise.
---
 crates/weavepy-vm/src/lib.rs | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index d39e730f..4362783f 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -20916,7 +20916,10 @@ impl Interpreter {
             | InlineResume::NextCall
             | InlineResume::NextDefault
             | InlineResume::SendCall => {
-                if self.core_iter_collapse(sw, pc, mode) == SendHop::Collapsed {
+                // SAFETY: see `CoreSwitch`: the running activation is synced.
+                if Self::iter_may_collapse(unsafe { &*sw.cur }, mode)
+                    && self.core_iter_collapse(sw, pc, mode) == SendHop::Collapsed
+                {
                     return true;
                 }
                 if !self.core_gen_resume_one(sw, pc, mode) {
@@ -20947,6 +20950,27 @@ impl Interpreter {
         true
     }
 
+    /// Whether the generator a `FOR_ITER`, `next()` or `send(None)` call
+    /// (`mode`) atop `frame`'s stack resumes is a collapsed chain's level
+    /// or suspended in a `yield from` (see [`Self::core_iter_collapse`]):
+    /// a cheap first look, so most generators skip that call.
+    #[inline(always)]
+    fn iter_may_collapse(frame: &Frame, mode: InlineResume) -> bool {
+        let Some(g) = Self::resumed_gen(frame, mode) else {
+            return false;
+        };
+        if g.chain.get() != 0 {
+            return true;
+        }
+        // SAFETY: a read between instructions; the view ends here.
+        match unsafe { g.state_raw().peek() } {
+            Some(GeneratorState::Suspended(gf)) => {
+                yield_from_resend(&gf.code.instructions, gf.pc as usize).is_some()
+            }
+            _ => false,
+        }
+    }
+
     /// [`Self::core_gen_resume`]'s `FOR_ITER` or `next()` (`mode`) at `pc`
     /// of a generator suspended in a `yield from`: the generator becomes
     /// the outermost level of a collapsed chain, as the delegate of a

From 5cb1fc3716301e289ed1808c2d8c45660493c47d Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 18:41:20 -0700
Subject: [PATCH 137/173] perf: serve module attributes, __dict__, and super()
 arms from site caches

Module attributes: a LOAD_ATTR site remembers the module dict's stamp and
the entry's position, and compiled code reads the value in line while the
stamp stands (CPython's LOAD_ATTR_MODULE), instead of decoding the inline
cache and comparing names in two helpers.

`inst.__dict__`: a site remembers that the receiver's class resolves the
name to its plain dict member, so the read no longer falls through every
leaf lane to the full lookup.

A `__getattr__` hook on a plain class whose default lookup certainly
misses is called without building the AttributeError the default lookup
would raise (CPython's slot_tp_getattr_hook suppresses it too). The
rescue's builtin slot-wrapper walk memoizes its negative answer per class
version.

The explicit `super(C, obj).name` form takes the same direct paths as the
implicit one, and a native `__init__` on a plain instance (object.__init__
through `super().__init__()`) runs its body directly instead of through
the full call dispatch.

Measured (instructions per iteration, host, PYTHONHASHSEED=0):
  5 module attribute reads    1970 -> 1033 (CPython 764)
  2x o.__dict__               5314 ->  785 (CPython 693)
  o.zz via __getattr__        8129 -> 5674 (CPython 1086)
  super().__init__() reaching object.__init__   7690 -> 3693 (CPython 2113)
  super(B, cls).__new__(cls)  17371 -> 12369 (CPython 2797)
Suite geomean (ins_suite.py) 1.420x -> 1.416x CPython; deepcopy -2.0%,
logging_mix, stdlib_utils and graph_search -1.4%.
---
 crates/weavepy-vm/src/frame_jit.rs         | 216 +++++++++++++---
 crates/weavepy-vm/src/lib.rs               | 219 ++++++++++++++--
 crates/weavepy-vm/src/types.rs             |   3 +
 tests/regrtest/test_attribute_slow_arms.py | 279 +++++++++++++++++++++
 4 files changed, 654 insertions(+), 63 deletions(-)
 create mode 100644 tests/regrtest/test_attribute_slow_arms.py

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 20e0ff18..f267e823 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -930,6 +930,10 @@ struct Tags {
     dict: u8,
     set: u8,
     str: u8,
+    /// A module's tag, and where its dict's data is (see
+    /// [`module_layout`]).
+    module: u8,
+    module_layout: Option<(i32, i32)>,
     /// The heap variants whose payload word is an `Rc`'s allocation, its
     /// strong count the allocation's first word (a clone is an increment
     /// there): instances, lists, dicts and classes, as far as measured.
@@ -1029,6 +1033,8 @@ fn tags() -> Option<Tags> {
         dict: tag(&Object::new_dict()),
         set: tag(&Object::new_set()),
         str: tag(&Object::from_str("")),
+        module: tag(&sample_module()),
+        module_layout: module_layout(),
         rc_counted_hdr: if cfg!(debug_assertions) {
             0
         } else {
@@ -1135,6 +1141,38 @@ fn tuple_layout() -> Option<(i32, i32)> {
         .then(|| Some((i32::try_from(len * 8).ok()?, i32::try_from(items).ok()?)))?
 }
 
+/// A private module to measure against (never seen by Python code).
+fn sample_module() -> Object {
+    Object::Module(Rc::new(crate::object::PyModule {
+        name: String::new(),
+        filename: None,
+        dict: Rc::new(crate::sync::RefCell::new(crate::object::DictData::default())),
+    }))
+}
+
+/// Where a module's dict is, read off a sample module (`None` if it isn't
+/// the shape the code assumes): the byte offset, from its payload word, of
+/// its dict handle (a word), and from that handle's value, of the dict's
+/// data (where its stamp and entries are). A module's dict is fixed for
+/// its lifetime.
+fn module_layout() -> Option<(i32, i32)> {
+    let m = sample_module();
+    // SAFETY: every `Object` is 16 bytes; a module's second word addresses
+    // its payload.
+    let word = unsafe { *std::ptr::from_ref(&m).cast::<u64>().add(1) } as usize;
+    let Object::Module(pm) = &m else {
+        return None;
+    };
+    let field = std::ptr::from_ref(&pm.dict) as usize;
+    let handle_at = field.checked_sub(word)?;
+    // SAFETY: the dict handle is one word inside the live sample.
+    let handle = unsafe { std::ptr::read(field as *const usize) };
+    let cell: &crate::sync::RefCell<crate::object::DictData> = &pm.dict;
+    let data_at = (cell.as_ptr() as usize).checked_sub(handle)?;
+    (std::mem::size_of::<Rc<crate::sync::RefCell<crate::object::DictData>>>() == 8)
+        .then(|| Some((i32::try_from(handle_at).ok()?, i32::try_from(data_at).ok()?)))?
+}
+
 /// Whether cloning `o` (whose payload word is `w`) adds one to the word at
 /// `w` and nothing else measurable, and dropping the clone takes it away.
 fn counted_at_payload(o: &Object, w: u64) -> bool {
@@ -3666,10 +3704,48 @@ impl ModCache {
 /// A `LOAD_ATTR` site's `__slots__` member shortcut: the class version and
 /// the member's position the site's cache last proved (`ver` 0 empty),
 /// read without decoding the site's cache.
+///
+/// A module receiver's read keeps its own pair: the module dict's stamp
+/// when the site last read it (`0` empty: no dict has that stamp) and
+/// the address of the entry's value, which compiled code reads in line
+/// while the stamp stands (as [`GlobalCache`] does a global's).
 #[derive(Default)]
+#[repr(C)]
 pub(crate) struct SlotCache {
     ver: std::cell::Cell<u64>,
     idx: std::cell::Cell<u32>,
+    mstamp: std::cell::Cell<u64>,
+    mvalue: std::cell::Cell<u64>,
+}
+
+const SC_MSTAMP: i32 = std::mem::offset_of!(SlotCache, mstamp) as i32;
+const SC_MVALUE: i32 = std::mem::offset_of!(SlotCache, mvalue) as i32;
+
+/// Remember in the `LOAD_ATTR` site's `cache` the entry of `m`'s dict the
+/// site's stamp (see `MODULE_ENTRY`) just proved, for compiled code's
+/// in-line read.
+fn mod_attr_note(
+    ext: &CodeConstObjects,
+    m: &crate::object::PyModule,
+    pc: usize,
+    cache: &SlotCache,
+) {
+    let Some(slot) = ext.stamp_slot(pc) else {
+        return;
+    };
+    let [stamp, tag, idx] = slot.get();
+    if tag != crate::MODULE_ENTRY {
+        return;
+    }
+    // SAFETY: a read between two instructions, running no code.
+    if let Some(d) = unsafe { m.dict.peek() } {
+        if d.mutation_stamp() == stamp {
+            if let Some((_, v)) = d.get_index(idx as usize) {
+                cache.mstamp.set(stamp);
+                cache.mvalue.set(std::ptr::from_ref(v) as u64);
+            }
+        }
+    }
 }
 
 /// The `__slots__` member `recv` holds for the `LOAD_ATTR` at `pc`, through
@@ -3775,8 +3851,16 @@ unsafe extern "C" fn h_stack_attr(
                 }
             },
             Object::Type(cls) if Rc::strong_count(cls) > 1 => type_attr(code, ext, cls, pc, arg),
-            Object::Module(m) => Interpreter::core_module_attr(code, m, pc, arg)
-                .or_else(|| Interpreter::leaf_fused_local_attr(code, recv, pc, arg)),
+            Object::Module(m) => {
+                let Some(v) = Interpreter::core_module_attr(code, m, pc, arg)
+                    .or_else(|| Interpreter::leaf_fused_local_attr(code, recv, pc, arg))
+                else {
+                    return 1;
+                };
+                mod_attr_note(ext, m, pc, &*cache);
+                crate::drop_hot(std::mem::replace(&mut *at, v));
+                return 0;
+            }
             Object::Code(_) | Object::Frame(_) => {
                 let Some(v) = code
                     .names
@@ -3854,9 +3938,17 @@ unsafe extern "C" fn h_local_attr(
             Object::Instance(inst) if inst.cls_raw().native_kind.get() != 0 => {
                 Interpreter::core_native_field(Some(ext), inst, arg, pc, code.instructions.len())
             }
-            Object::Instance(_) | Object::Module(_) => {
-                Interpreter::core_local_attr(Some(ext), code, recv, pc, arg)
+            Object::Module(m) => {
+                let Some(v) = Interpreter::core_module_attr(code, m, pc, arg)
+                    .or_else(|| Interpreter::core_local_attr(Some(ext), code, recv, pc, arg))
+                else {
+                    return 1;
+                };
+                mod_attr_note(ext, m, pc, &*cache);
+                dst.write(v);
+                return 0;
             }
+            Object::Instance(_) => Interpreter::core_local_attr(Some(ext), code, recv, pc, arg),
             Object::Type(cls) => type_attr(code, ext, cls, pc, arg),
             // A frame's or code object's plain field (a live frame's line
             // needs the full lane, which makes its `lasti` current).
@@ -10376,6 +10468,50 @@ impl<'a> Lower<'a> {
     /// [`h_local_attr`] / [`h_stack_attr`]. The value goes onto the stack
     /// (a scalar copied, anything else with a reference taken); a stack
     /// receiver is released after. A miss is the core loop's.
+    /// An in-line attribute read's value at `v` into the free slot `dst`
+    /// (stack slot `slot`) in place of the receiver `item`, then on to
+    /// `done`: a local receiver stays, a stack receiver is released first
+    /// (a declined release leaves through `exit`).
+    fn attr_value_out(
+        &mut self,
+        item: Item,
+        dst: Value,
+        slot: usize,
+        v: Value,
+        exit: Block,
+        done: Block,
+    ) {
+        match item {
+            // The receiver is the local's: the value just takes a copy.
+            Item::Local(_) => {
+                self.copy_value(dst, v);
+                self.b.ins().jump(done, &[]);
+            }
+            _ => {
+                // The receiver is the stack's: the value goes above it
+                // (copied) until the receiver is released, then down.
+                let above = self.slot_addr(slot + 1);
+                self.copy_value(above, v);
+                let r = self
+                    .call(h_pop as *const () as usize, &[dst], true)
+                    .expect("returns");
+                let undo = self.b.create_block();
+                let released = self.b.create_block();
+                self.b.ins().brif(r, undo, &[], released, &[]);
+                // Declined: the copy goes, and the core loop takes the
+                // instruction from the start.
+                self.b.switch_to_block(undo);
+                // (A fresh copy has another owner: its release is a plain
+                // decrement.)
+                self.call(h_pop as *const () as usize, &[above], true);
+                self.b.ins().jump(exit, &[]);
+                self.b.switch_to_block(released);
+                self.copy16(dst, above);
+                self.b.ins().jump(done, &[]);
+            }
+        }
+    }
+
     fn load_attr(&mut self, pc: usize, _arg: u32) -> bool {
         // A property's getter may run here (see `h_property`), which needs
         // the whole stack on the frame's.
@@ -10397,8 +10533,41 @@ impl<'a> Lower<'a> {
         let dst = self.slot_addr(slot);
         let miss = self.b.create_block();
         let done = self.b.create_block();
+        let cache: Box<SlotCache> = Box::default();
+        let cache_at = std::ptr::from_ref::<SlotCache>(&cache) as i64;
+        self.slot_caches.push(cache);
+        let module_site = matches!(
+            self.code.caches.get(pc as u32),
+            weavepy_compiler::InlineCache::LoadAttrModule { .. }
+        );
         // (A site with no field shortcut yet goes straight to the helper.)
         match (self.layout, self.field_slots.get(pc)) {
+            // A module attribute (CPython's `LOAD_ATTR_MODULE`): the value
+            // the site's helper last read, while the module's dict is
+            // exactly as it was then (its stamp moves with every key
+            // change; a value stored in place stays at its address).
+            _ if module_site && self.tags.module_layout.is_some() => {
+                let (handle_at, data_at) = self.tags.module_layout.expect("checked");
+                let ptr = self.ptr;
+                let otag = self.tag_at(at);
+                let not_mod =
+                    self.b
+                        .ins()
+                        .icmp_imm(IntCC::NotEqual, otag, i64::from(self.tags.module));
+                self.branch_out(not_mod, miss);
+                let m = self.b.ins().load(ptr, FLAGS, at, 8);
+                let handle = self.b.ins().load(ptr, FLAGS, m, handle_at);
+                let stamp = self
+                    .b
+                    .ins()
+                    .load(types::I64, FLAGS, handle, data_at + DICT_STAMP);
+                let c = self.b.ins().iconst(ptr, cache_at);
+                let want = self.b.ins().load(types::I64, FLAGS, c, SC_MSTAMP);
+                let stale = self.b.ins().icmp(IntCC::NotEqual, stamp, want);
+                self.branch_out(stale, miss);
+                let v = self.b.ins().load(ptr, FLAGS, c, SC_MVALUE);
+                self.attr_value_out(item, dst, slot, v, exit, done);
+            }
             (Some(l), Some(cache_slot)) if cache_slot.get().0 != 0 => {
                 let l = *l;
                 let otag = self.tag_at(at);
@@ -10486,37 +10655,7 @@ impl<'a> Lower<'a> {
                     let v = self.b.ins().iadd(block, off);
                     self.b.ins().iadd_imm(v, i64::from(l.split_values))
                 };
-                match item {
-                    // The receiver is the local's: the value just takes a
-                    // copy.
-                    Item::Local(_) => {
-                        self.copy_value(dst, v);
-                        self.b.ins().jump(done, &[]);
-                    }
-                    _ => {
-                        // The receiver is the stack's: the value goes above
-                        // it (copied) until the receiver is released, then
-                        // down.
-                        let above = self.slot_addr(slot + 1);
-                        self.copy_value(above, v);
-                        let r = self
-                            .call(h_pop as *const () as usize, &[dst], true)
-                            .expect("returns");
-                        let undo = self.b.create_block();
-                        let released = self.b.create_block();
-                        self.b.ins().brif(r, undo, &[], released, &[]);
-                        // Declined: the copy goes, and the core loop takes
-                        // the instruction from the start.
-                        self.b.switch_to_block(undo);
-                        // (A fresh copy has another owner: its release is a
-                        // plain decrement.)
-                        self.call(h_pop as *const () as usize, &[above], true);
-                        self.b.ins().jump(exit, &[]);
-                        self.b.switch_to_block(released);
-                        self.copy16(dst, above);
-                        self.b.ins().jump(done, &[]);
-                    }
-                }
+                self.attr_value_out(item, dst, slot, v, exit, done);
             }
             _ => {
                 self.b.ins().jump(miss, &[]);
@@ -10527,12 +10666,7 @@ impl<'a> Lower<'a> {
         let code = self.code_ptr();
         let ext = self.ext_ptr();
         let pcv = self.b.ins().iconst(types::I64, pc as i64);
-        let cache: Box<SlotCache> = Box::default();
-        let c = self
-            .b
-            .ins()
-            .iconst(self.ptr, std::ptr::from_ref::<SlotCache>(&cache) as i64);
-        self.slot_caches.push(cache);
+        let c = self.b.ins().iconst(self.ptr, cache_at);
         let r = match item {
             Item::Local(_) => self.call(
                 h_local_attr as *const () as usize,
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 4362783f..38943a46 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -15401,6 +15401,12 @@ impl Interpreter {
                 {
                     return Some(true);
                 }
+                // `super().__init__(...)` reaching `object.__init__` (or
+                // another native `__init__`) on a plain instance: the
+                // dispatch's only `__init__` case is a dict payload's.
+                if native_init_call(b, recv) {
+                    return Some(false);
+                }
                 // The names below are dispatched by the interpreter, which
                 // knows them on the builtin kinds' receivers.
                 builtins::method_memo_tag(recv)?;
@@ -18223,7 +18229,8 @@ impl Interpreter {
         base: *mut Object,
         len: usize,
     ) -> Option<usize> {
-        if ins.arg & 2 != 0 || len < 3 {
+        // (Either form; see `load_super_attr_direct`.)
+        if len < 3 {
             return None;
         }
         // SAFETY: `len >= 3`.
@@ -23279,13 +23286,61 @@ impl Interpreter {
     }
 
     /// A module attribute through the `LOAD_ATTR` site's cached index:
-    /// the value, cloned, or `None` for the helper's full path.
+    /// the value, cloned, or `None` for the helper's full path. A site
+    /// remembers the module dict's stamp and the entry's position once a
+    /// read proves them (see [`MODULE_ENTRY`]), so a warm read is a stamp
+    /// compare and an indexed load (CPython's `LOAD_ATTR_MODULE`).
     #[inline]
     fn core_module_attr(
         code: &CodeObject,
         m: &crate::object::PyModule,
         pc: usize,
         name_idx: u32,
+    ) -> Option<Object> {
+        // SAFETY: a read between two instructions (see `GilCell::peek`).
+        let dict = unsafe { m.dict.peek() }?;
+        if let Some(slot) = code_vm_ext(code).and_then(|ext| ext.stamp_slot(pc)) {
+            let [stamp, tag, idx] = slot.get();
+            if tag == MODULE_ENTRY && stamp == dict.mutation_stamp() {
+                if let Some((_, v)) = dict.get_index(idx as usize) {
+                    return Some(Self::clone_operand(v));
+                }
+            }
+        }
+        Self::core_module_attr_indexed(code, m, pc, name_idx)
+    }
+
+    /// [`Self::core_module_attr`] through the site's inline cache, which
+    /// on a hit also fills the site's [`MODULE_ENTRY`] stamp.
+    #[inline(never)]
+    fn core_module_attr_indexed(
+        code: &CodeObject,
+        m: &crate::object::PyModule,
+        pc: usize,
+        name_idx: u32,
+    ) -> Option<Object> {
+        let v = Self::core_module_attr_cached(code, m, pc, name_idx)?;
+        // SAFETY: as above (the read above ran no code).
+        if let (Some(dict), Some(slot)) =
+            (unsafe { m.dict.peek() }, code_stamp_slot(code, pc as u32))
+        {
+            if let weavepy_compiler::InlineCache::LoadAttrModule { key_idx, .. } =
+                code.caches.get(pc as u32)
+            {
+                slot.set([dict.mutation_stamp(), MODULE_ENTRY, u64::from(key_idx)]);
+            }
+        }
+        Some(v)
+    }
+
+    /// [`Self::core_module_attr_indexed`]'s read of the site's inline
+    /// cache.
+    #[inline]
+    fn core_module_attr_cached(
+        code: &CodeObject,
+        m: &crate::object::PyModule,
+        pc: usize,
+        name_idx: u32,
     ) -> Option<Object> {
         use weavepy_compiler::InlineCache as IC;
         let IC::LoadAttrModule { module_id, key_idx } = code.caches.get(pc as u32) else {
@@ -23373,7 +23428,10 @@ impl Interpreter {
         }
         // `leaf_fused_local_attr`, over the decoded cache.
         if matches!(cache, IC::Empty) {
-            return None;
+            return match local {
+                Object::Instance(inst) => inst_dict_attr(code, inst, attr_pc, name_idx),
+                _ => None,
+            };
         }
         match local {
             Object::Instance(i) if i.cls_raw().native_kind.get() == 0 => {}
@@ -23915,7 +23973,10 @@ impl Interpreter {
     ) -> Option<Object> {
         use weavepy_compiler::InlineCache as IC;
         if matches!(code.caches.get(attr_pc as u32), IC::Empty) {
-            return None;
+            return match local {
+                Object::Instance(inst) => inst_dict_attr(code, inst, attr_pc, name_idx),
+                _ => None,
+            };
         }
         match local {
             Object::Instance(i) if i.cls_raw().native_kind.get() == 0 => {}
@@ -25466,10 +25527,7 @@ impl Interpreter {
                     // pushed in the elided-method shape (function, self)
                     // — the `CALL` folds `self` in — or bound, without the
                     // super proxy the full handler builds. Stack (top-
-                    // down): self, class, global `super`.
-                    if ins.arg & 2 != 0 {
-                        break;
-                    }
+                    // down): self, class, global `super` (either form).
                     let n = stack.len();
                     if n < 3 || !is_super_callable(&stack[n - 3]) {
                         break;
@@ -28209,6 +28267,9 @@ impl Interpreter {
                 return Some(v);
             }
         }
+        if let Some(v) = inst_dict_attr(code, inst, cache_pc as usize, name_idx) {
+            return Some(v);
+        }
         let poly = code_attr_poly(code, cache_pc);
         let ver = inst.cls_raw().attr_version.get();
         // A never-specialized site resolves once and takes the indexed
@@ -35962,23 +36023,21 @@ impl Interpreter {
                 &self.builtins.clone(),
             )
         } else {
+            // The default lookup would only raise for the hook to catch:
+            // call the hook without building that `AttributeError` (CPython's
+            // `slot_tp_getattr_hook` suppresses it the same way).
+            let hook = inst.cls_raw().dunder(crate::types::Dunder::GetAttr).present();
+            if hook && default_lookup_misses(instance_obj, name, true) {
+                if let Some(getattr) = inst.cls().lookup("__getattr__") {
+                    return self.call_getattr_hook(instance_obj, getattr, name);
+                }
+            }
             self.load_attr_instance_default(inst, instance_obj, name)
         };
         match result {
             Err(e) if self.is_attribute_error(&e) => {
                 if let Some(getattr) = inst.cls().lookup("__getattr__") {
-                    // `dispatch` so a descriptor `__getattr__` also honours
-                    // its `__get__` (test_descr test_getattr_hooks).
-                    let bound = Object::BoundMethod(Rc::new(BoundMethod::dispatch(
-                        instance_obj.clone(),
-                        getattr,
-                    )));
-                    self.call(
-                        &bound,
-                        &[Object::from_str(name)],
-                        &[],
-                        &self.builtins.clone(),
-                    )
+                    self.call_getattr_hook(instance_obj, getattr, name)
                 } else {
                     Err(e)
                 }
@@ -35987,6 +36046,27 @@ impl Interpreter {
         }
     }
 
+    /// `type(obj).__getattr__(obj, name)`, the hook `getattr` resolved.
+    fn call_getattr_hook(
+        &mut self,
+        instance_obj: &Object,
+        getattr: Object,
+        name: &str,
+    ) -> Result<Object, RuntimeError> {
+        // `dispatch` so a descriptor `__getattr__` also honours its
+        // `__get__` (test_descr test_getattr_hooks).
+        let bound = Object::BoundMethod(Rc::new(BoundMethod::dispatch(
+            instance_obj.clone(),
+            getattr,
+        )));
+        self.call(
+            &bound,
+            &[Object::from_str(name)],
+            &[],
+            &self.builtins.clone(),
+        )
+    }
+
     /// A user-defined `__getattribute__` override, if any. Returns `None`
     /// when the class uses the default `object.__getattribute__` (resolved as
     /// the `.object_getattribute` sentinel installed on `object`).
@@ -36117,7 +36197,10 @@ impl Interpreter {
         frame: &mut Frame,
         arg: u32,
     ) -> Result<Option<Object>, RuntimeError> {
-        if arg & 2 != 0 || crate::trace::any_observers_active() {
+        // (Both forms: the explicit `super(C, obj)` puts its arguments
+        // where the implicit one puts the `__class__` cell and the first
+        // argument, and CPython specializes either.)
+        if crate::trace::any_observers_active() {
             return Ok(None);
         }
         let n = frame.stack.len();
@@ -68997,6 +69080,15 @@ mod builtin_type_misses {
 /// `getattr` default or `hasattr` then needs no exception at all. Dunder
 /// names, which the lookup special-cases, never qualify.
 fn attr_certainly_missing(obj: &Object, name: &str) -> bool {
+    default_lookup_misses(obj, name, false)
+}
+
+/// Whether the default `object.__getattribute__` lookup of `obj.name`
+/// certainly ends in its `AttributeError` without running any code (see
+/// [`attr_certainly_missing`]), ignoring a `__getattr__` hook on the class
+/// when `hooked` (the caller then calls the hook itself, as CPython's
+/// `slot_tp_getattr_hook` does after a suppressed generic lookup).
+fn default_lookup_misses(obj: &Object, name: &str, hooked: bool) -> bool {
     use crate::types::Dunder;
     let Object::Instance(inst) = obj else {
         return false;
@@ -69009,11 +69101,17 @@ fn attr_certainly_missing(obj: &Object, name: &str) -> bool {
     if cls.native_kind.get() != 0 {
         return false;
     }
+    // (A builtin class's hook, a weak proxy's among them, may stand in for
+    // special cases of the default lookup.)
+    let hook = cls.dunder(Dunder::GetAttr).present();
+    if hook && (!hooked || cls.flags.is_builtin) {
+        return false;
+    }
     // The class's part of the verdict depends only on its attributes
     // (memoized until it or a base changes).
     let class_missing = cls.memo_verdict(crate::types::Verdict::AttrMissing, name, || {
         if !cls.dunder(Dunder::GetAttribute).object_owner()
-            || cls.dunder(Dunder::GetAttr).present()
+            || cls.is_super_proxy_type()
             || cls.lookup(name).is_some()
         {
             return false;
@@ -69075,6 +69173,18 @@ pub(crate) fn builtin_slot_wrapper(ty: &Rc<TypeObject>, name: &str) -> Option<Ob
             _ => Object::None,
         });
     }
+    // A name no builtin base supplies (the common case: the rescue of every
+    // failed instance lookup asks) is settled once per class version.
+    if ty.memo_verdict(crate::types::Verdict::NoSlotWrapper, name, || {
+        builtin_slot_wrapper_walk(ty, name).is_none()
+    }) {
+        return None;
+    }
+    builtin_slot_wrapper_walk(ty, name)
+}
+
+/// [`builtin_slot_wrapper`]'s walk of `ty`'s builtin bases.
+fn builtin_slot_wrapper_walk(ty: &Rc<TypeObject>, name: &str) -> Option<Object> {
     for base in ty.mro.borrow().iter() {
         if !base.flags.is_builtin {
             continue;
@@ -73332,6 +73442,9 @@ fn exception_init_call(b: &crate::object::BuiltinFn, recv: &Object) -> bool {
 #[inline]
 fn lane_recv_ok(b: &crate::object::BuiltinFn, recv: &Object, via_call: bool) -> bool {
     b.binds_instance
+        // (A site settled on a plain instance's native `__init__` may see
+        // a dict payload later, which the dispatch routes.)
+        && (via_call || b.name != "__init__" || native_init_call(b, recv))
         && (!via_call
             || builtins::method_memo_tag(recv).is_some()
             || exception_init_call(b, recv)
@@ -73339,6 +73452,18 @@ fn lane_recv_ok(b: &crate::object::BuiltinFn, recv: &Object, via_call: bool) ->
                 && matches!(recv, Object::Instance(_))))
 }
 
+/// Whether `b` called on `recv` is a native `__init__` (`object.__init__`,
+/// typically through `super().__init__()`) on a plain instance: no native
+/// payload (a dict's routes through the dispatch) and not an exception
+/// (see [`exception_init_call`]), so its body runs directly.
+fn native_init_call(b: &crate::object::BuiltinFn, recv: &Object) -> bool {
+    b.name == "__init__"
+        && b.call_kw.is_none()
+        && matches!(recv, Object::Instance(i) if i.native.get().is_none()
+            && i.c_body.get() == 0
+            && !i.cls_raw().flags.is_exception)
+}
+
 /// The builtin function a bound method of one wraps, as its own callable.
 #[cold]
 fn bm_builtin(callable: &Object) -> Object {
@@ -74965,6 +75090,56 @@ const LAID_OUT_SLOT: u64 = 0x5ca1_a770_0000_0006;
 /// [`Interpreter::core_delete_attr`]).
 const DELETE_OWN_ATTR: u64 = 0x5ca1_a770_0000_0007;
 
+/// A `LOAD_ATTR` site's stamp tag for a module attribute: the first word
+/// is the module dict's mutation stamp, the bits the entry's position.
+/// A stamp names one key layout of one dict (values stored in place keep
+/// it), so while it matches, that position holds the site's name.
+const MODULE_ENTRY: u64 = 0x5ca1_a770_0000_0008;
+
+/// A `LOAD_ATTR` site's stamp tag for `inst.__dict__` where the class (at
+/// that version) resolves the name to its own `__dict__` member, a data
+/// descriptor serving the instance dict (CPython's `subtype_dict`).
+const INST_DICT: u64 = 0x5ca1_a770_0000_0009;
+
+/// `inst.__dict__` for the `LOAD_ATTR` site at `pc` (name `name_idx`):
+/// through the site's [`INST_DICT`] stamp, or, for a class that resolves
+/// `__dict__` to its plain dict member under the default attribute
+/// protocol, the instance dict, after which the site remembers the class.
+/// `None` for any other name or shape. Runs no code (the dict may be
+/// created).
+#[inline(never)]
+fn inst_dict_attr(
+    code: &CodeObject,
+    inst: &PyInstance,
+    pc: usize,
+    name_idx: u32,
+) -> Option<Object> {
+    if code.names.get(name_idx as usize)?.as_str() != "__dict__"
+        || inst.native.get().is_some()
+        || inst.c_body.get() != 0
+    {
+        return None;
+    }
+    let cls = inst.cls_raw();
+    let ver = cls.attr_version.get();
+    let warm = code_vm_ext(code)
+        .and_then(|ext| ext.stamp_slot(pc))
+        .is_some_and(|slot| slot.get()[..2] == [ver, INST_DICT]);
+    if !warm {
+        if !cls.has_managed_dict()
+            || cls.native_kind.get() != 0
+            || cls.c_ext_ptr.get() != 0
+            || !Interpreter::default_getattribute(cls)
+            || cls.is_super_proxy_type()
+            || !matches!(cls.lookup("__dict__"), Some(Object::SlotDescriptor(s)) if s.name == "__dict__")
+        {
+            return None;
+        }
+        code_stamp_slot(code, pc as u32)?.set([ver, INST_DICT, 0]);
+    }
+    Some(Object::Dict(inst.dict_shared()))
+}
+
 /// The named tuple field a `LOAD_ATTR` site's stamp remembers for
 /// `inst`'s class at its current version (see [`TUPLE_FIELD`]): the item
 /// of `inst`'s tuple, or `None` (an index out of range is the
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 6979a047..f7fd5501 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -159,6 +159,9 @@ pub(crate) enum Verdict {
     /// A builtin class's own dict has no class method `name` (see
     /// `Interpreter::leaf_load_type_attr`).
     NoNativeClassMethod = 3,
+    /// No builtin base on the class's MRO supplies a slot wrapper or a
+    /// type-dict entry `name` (see `builtin_slot_wrapper`).
+    NoSlotWrapper = 4,
 }
 
 pub struct AttrVersion(std::sync::atomic::AtomicU64);
diff --git a/tests/regrtest/test_attribute_slow_arms.py b/tests/regrtest/test_attribute_slow_arms.py
new file mode 100644
index 00000000..f8e9a9d6
--- /dev/null
+++ b/tests/regrtest/test_attribute_slow_arms.py
@@ -0,0 +1,279 @@
+"""The slower arms of attribute resolution keep their exact semantics.
+
+Module attributes, `inst.__dict__`, a `__getattr__` hook reached without
+the default lookup's `AttributeError`, both forms of `super().name`, and a
+native `__init__` reached through `super()` are each served by a cached or
+direct path once warm. Changes to the module, the class, or the receiver's
+shape must be seen at once.
+"""
+
+import types
+import unittest
+
+
+N = 300
+
+
+class Plain:
+    def __init__(self):
+        self.a = 1
+
+
+class Hooked:
+    def __init__(self):
+        self.a = 1
+
+    def __getattr__(self, name):
+        if name.startswith("x"):
+            return name.upper()
+        raise AttributeError(f"hooked {name}")
+
+
+def read_module_attr(m, n):
+    out = []
+    for _ in range(n):
+        out.append(m.value)
+    return out
+
+
+def read_dict(o, n):
+    out = None
+    for _ in range(n):
+        out = o.__dict__
+    return out
+
+
+def read_hooked(o, n):
+    out = []
+    for _ in range(n):
+        out.append(o.xy)
+    return out
+
+
+class ModuleAttrTest(unittest.TestCase):
+    def test_rebinding_and_deletion(self):
+        m = types.ModuleType("m")
+        m.value = 1
+        m.other = 2
+        self.assertEqual(read_module_attr(m, N), [1] * N)
+        m.value = 3
+        self.assertEqual(read_module_attr(m, 5), [3] * 5)
+        # Keys move: the site must not read a stale position.
+        del m.other
+        m.extra = 4
+        self.assertEqual(read_module_attr(m, 5), [3] * 5)
+        del m.value
+        with self.assertRaises(AttributeError):
+            read_module_attr(m, 1)
+        m.value = 7
+        self.assertEqual(read_module_attr(m, 5), [7] * 5)
+
+    def test_another_module_at_the_same_site(self):
+        a = types.ModuleType("a")
+        b = types.ModuleType("b")
+        a.value, b.pad, b.value = "a", 0, "b"
+        self.assertEqual(read_module_attr(a, N), ["a"] * N)
+        self.assertEqual(read_module_attr(b, 3), ["b"] * 3)
+        self.assertEqual(read_module_attr(a, 3), ["a"] * 3)
+
+    def test_module_getattr(self):
+        m = types.ModuleType("m")
+        m.__getattr__ = lambda name: "dyn:" + name
+        self.assertEqual(read_module_attr(m, N), ["dyn:value"] * N)
+        m.value = 1
+        self.assertEqual(read_module_attr(m, 3), [1] * 3)
+
+
+class InstanceDictTest(unittest.TestCase):
+    def test_identity_and_updates(self):
+        o = Plain()
+        d = read_dict(o, N)
+        self.assertIs(d, o.__dict__)
+        self.assertEqual(d, {"a": 1})
+        o.b = 2
+        self.assertEqual(read_dict(o, 3), {"a": 1, "b": 2})
+
+    def test_class_level_override_is_seen(self):
+        o = Plain()
+        read_dict(o, N)
+        Plain.__dict__  # (the class's own mapping proxy is untouched)
+        try:
+            Plain.__dict__ = property(lambda self: "custom")
+        except (AttributeError, TypeError):
+            pass
+
+        class Sub(Plain):
+            @property
+            def __dict__(self):
+                return "custom"
+
+        self.assertEqual(read_dict(Sub(), 3), "custom")
+        self.assertEqual(read_dict(o, 3), {"a": 1})
+
+    def test_slots_class_has_no_dict(self):
+        class S:
+            __slots__ = ("a",)
+
+        read_dict(Plain(), N)
+        with self.assertRaises(AttributeError):
+            read_dict(S(), 1)
+
+    def test_getattribute_override(self):
+        class G(Plain):
+            def __getattribute__(self, name):
+                if name == "__dict__":
+                    return "intercepted"
+                return object.__getattribute__(self, name)
+
+        read_dict(Plain(), N)
+        self.assertEqual(read_dict(G(), 3), "intercepted")
+
+
+class GetattrHookTest(unittest.TestCase):
+    def test_hook_values_and_errors(self):
+        o = Hooked()
+        self.assertEqual(read_hooked(o, N), ["XY"] * N)
+        with self.assertRaises(AttributeError) as cm:
+            o.missing
+        self.assertEqual(str(cm.exception), "hooked missing")
+        self.assertEqual(cm.exception.name, "missing")
+        self.assertIs(cm.exception.obj, o)
+        self.assertFalse(hasattr(o, "missing"))
+        self.assertEqual(getattr(o, "missing", 9), 9)
+
+    def test_real_attributes_win(self):
+        o = Hooked()
+        read_hooked(o, N)
+        o.xy = "own"
+        self.assertEqual(read_hooked(o, 3), ["own"] * 3)
+        del o.xy
+        Hooked.xy = "class"
+        try:
+            self.assertEqual(read_hooked(o, 3), ["class"] * 3)
+        finally:
+            del Hooked.xy
+        self.assertEqual(read_hooked(o, 3), ["XY"] * 3)
+
+    def test_data_descriptor_raising(self):
+        class D:
+            @property
+            def xy(self):
+                raise AttributeError("from property")
+
+            def __getattr__(self, name):
+                return "hook:" + name
+
+        self.assertEqual(read_hooked(D(), N), ["hook:xy"] * N)
+
+    def test_hook_added_and_removed(self):
+        class C:
+            pass
+
+        o = C()
+        with self.assertRaises(AttributeError):
+            read_hooked(o, 1)
+        C.__getattr__ = lambda self, name: 1
+        self.assertEqual(read_hooked(o, N), [1] * N)
+        del C.__getattr__
+        with self.assertRaises(AttributeError) as cm:
+            read_hooked(o, 1)
+        self.assertEqual(str(cm.exception), "'C' object has no attribute 'xy'")
+
+
+class Base:
+    def __new__(cls, *args):
+        return super().__new__(cls)
+
+    def __init__(self, x):
+        self.x = x
+
+    def m(self):
+        return "base"
+
+
+class Explicit(Base):
+    def __new__(cls, *args):
+        return super(Explicit, cls).__new__(cls)
+
+    def __init__(self, x):
+        super(Explicit, self).__init__(x * 2)
+
+    def m(self):
+        return "explicit+" + super(Explicit, self).m()
+
+
+class SuperFormsTest(unittest.TestCase):
+    def test_explicit_form(self):
+        for i in range(N):
+            e = Explicit(i)
+            self.assertEqual(e.x, i * 2)
+            self.assertEqual(e.m(), "explicit+base")
+
+    def test_explicit_form_errors(self):
+        def bad(cls, obj):
+            return super(cls, obj).m
+
+        for _ in range(N):
+            bad(Explicit, Explicit(1))
+        with self.assertRaises(TypeError):
+            bad(Explicit, Base(1))
+        with self.assertRaises(TypeError):
+            bad(1, Explicit(1))
+
+    def test_mro_change_is_seen(self):
+        class A:
+            def f(self):
+                return "A"
+
+        class B(A):
+            def f(self):
+                return super(B, self).f()
+
+        b = B()
+        for _ in range(N):
+            self.assertEqual(b.f(), "A")
+        A.f = lambda self: "A2"
+        self.assertEqual(b.f(), "A2")
+
+
+class ObjectInitTest(unittest.TestCase):
+    def test_object_init_through_super(self):
+        class O:
+            def __init__(self, x):
+                super().__init__()
+                self.x = x
+
+        for i in range(N):
+            self.assertEqual(O(i).x, i)
+
+    def test_object_init_excess_arguments(self):
+        class O:
+            def __init__(self, *args):
+                super().__init__(*args)
+
+        for _ in range(N):
+            O()
+        with self.assertRaises(TypeError) as cm:
+            O(1)
+        self.assertIn("object.__init__() takes exactly one argument", str(cm.exception))
+
+    def test_dict_payload_at_a_settled_site(self):
+        class B:
+            def __init__(self, *args):
+                super().__init__(*args)
+
+        class D(dict, B):
+            def __init__(self, *args):
+                dict.__init__(self, *args)
+                # `object.__init__` follows `B` in the MRO: the site sees
+                # a dict payload after settling on plain instances.
+                B.__init__(self)
+
+        for _ in range(N):
+            B()
+        d = D([("k", 1)])
+        self.assertEqual(d, {"k": 1})
+
+
+if __name__ == "__main__":
+    unittest.main()

From 712a6fe9f467c0703dc18bb54c9e5ab8fdc5a48e Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 19:47:33 -0700
Subject: [PATCH 138/173] perf: raise unset-slot and other native
 AttributeErrors cheaply

An unset `__slots__` member (pathlib's lazily computed `_str`, `_drv`,
... read under `try`/`except AttributeError`) cost far more than CPython:

- Its message formatted the class's qualified name through two hashed
  dict lookups and `format!` on every raise. The name is now memoized
  per class while its version, dict layout and C `tp_name` stand, and
  follows CPython's `%T` (no `__main__.` prefix, which WeavePy printed).
- A natively built AttributeError lays its `name` and `obj` fields out
  unset, so the attribute load stores them in place instead of
  converting the exception's slot storage.
- A `LOAD_ATTR_SLOT` site that meets an unset slot raises through the
  member descriptor and stays specialized (CPython deopts for that
  execution only); it used to drop to the cooldown cycle, so the hits
  that followed ran the generic lookup.
- Only a site cooling down after a failed load asks whether the
  attribute is certainly missing, and the `__getattr__` check after a
  failed default lookup reads the class's memoized dunder answer.

Measured (instructions per iteration, host, PYTHONHASHSEED=0):
  try: o._s except AttributeError (unset slot)  13408 -> 9970 (CPython 4983)
Suite geomean (ins_suite.py) 1.416x -> 1.414x CPython; stdlib_utils
-2.4%. Callgrind totals for decimal_telco and raytrace, which moved on
the host, are within 0.5% of the baseline.
---
 crates/weavepy-vm/src/error.rs             |  14 ++-
 crates/weavepy-vm/src/lib.rs               | 104 +++++++++++++++++---
 crates/weavepy-vm/src/types.rs             | 107 ++++++++++++++++++++-
 tests/regrtest/test_attribute_slow_arms.py |  71 ++++++++++++++
 4 files changed, 276 insertions(+), 20 deletions(-)

diff --git a/crates/weavepy-vm/src/error.rs b/crates/weavepy-vm/src/error.rs
index 33695bad..2616b541 100644
--- a/crates/weavepy-vm/src/error.rs
+++ b/crates/weavepy-vm/src/error.rs
@@ -275,7 +275,19 @@ pub fn name_error(message: impl Into<String>) -> RuntimeError {
 }
 
 pub fn attribute_error(message: impl Into<String>) -> RuntimeError {
-    RuntimeError::PyException(PyException::from_builtin("AttributeError", message))
+    // `name` and `obj` are laid out unset, so the attribute load the
+    // error propagates through stores them in place (see
+    // `attr_error_context_laid_out`).
+    let message: String = message.into();
+    let arg = (!message.is_empty()).then(|| Object::from_str(message));
+    let instance = crate::builtin_types::exception_from_parts_with(
+        crate::builtin_types::builtin_types()
+            .attribute_error
+            .clone(),
+        arg,
+        [("name", Object::Unbound), ("obj", Object::Unbound)],
+    );
+    RuntimeError::PyException(PyException::new(instance))
 }
 
 /// `AttributeError` carrying the structured `.name`/`.obj` fields
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 38943a46..32527530 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -34147,6 +34147,18 @@ impl Interpreter {
 
     fn load_attr(&mut self, obj: &Object, name: &str) -> Result<Object, RuntimeError> {
         let result = self.load_attr_inner(obj, name);
+        self.load_attr_finish(obj, name, result)
+    }
+
+    /// The end of [`Self::load_attr`] for the per-variant lookup's
+    /// `result`: the slot-wrapper rescue, and an `AttributeError`'s
+    /// `name`/`obj` context.
+    fn load_attr_finish(
+        &mut self,
+        obj: &Object,
+        name: &str,
+        result: Result<Object, RuntimeError>,
+    ) -> Result<Object, RuntimeError> {
         // CPython's `object.__getattribute__` final step: after the
         // per-variant lookup misses, resolve the name through
         // `type(obj)`'s MRO slot-wrapper table and bind it
@@ -34223,6 +34235,9 @@ impl Interpreter {
             if self.is_attribute_error(err) {
                 if let RuntimeError::PyException(pe) = err {
                     if let Object::Instance(inst) = &pe.instance {
+                        if attr_error_context_laid_out(inst, obj, name) {
+                            return result;
+                        }
                         let need = matches!(
                             crate::builtin_types::exc_attr(inst, "name"),
                             None | Some(Object::None)
@@ -36026,7 +36041,7 @@ impl Interpreter {
             // The default lookup would only raise for the hook to catch:
             // call the hook without building that `AttributeError` (CPython's
             // `slot_tp_getattr_hook` suppresses it the same way).
-            let hook = inst.cls_raw().dunder(crate::types::Dunder::GetAttr).present();
+            let hook = inst.cls().dunder(crate::types::Dunder::GetAttr).present();
             if hook && default_lookup_misses(instance_obj, name, true) {
                 if let Some(getattr) = inst.cls().lookup("__getattr__") {
                     return self.call_getattr_hook(instance_obj, getattr, name);
@@ -36036,10 +36051,12 @@ impl Interpreter {
         };
         match result {
             Err(e) if self.is_attribute_error(&e) => {
-                if let Some(getattr) = inst.cls().lookup("__getattr__") {
-                    self.call_getattr_hook(instance_obj, getattr, name)
-                } else {
-                    Err(e)
+                // (The memoized answer first: most classes have no hook.)
+                let cls = inst.cls();
+                let hook = cls.dunder(crate::types::Dunder::GetAttr).present();
+                match hook.then(|| cls.lookup("__getattr__")).flatten() {
+                    Some(getattr) => self.call_getattr_hook(instance_obj, getattr, name),
+                    None => Err(e),
                 }
             }
             other => other,
@@ -37303,13 +37320,19 @@ impl Interpreter {
                         None if slot.default.is_some() => Ok(slot.default.clone().unwrap()),
                         // CPython's member descriptor reports the
                         // owning type's *fully qualified* name here
-                        // (e.g. `pkg.mod.Cls`), unlike the generic
-                        // "has no attribute" path which uses `tp_name`.
-                        None => Err(attribute_error(format!(
-                            "'{}' object has no attribute '{}'",
-                            inst.cls().qualified_display_name(),
-                            slot.name
-                        ))),
+                        // (`%T`: e.g. `pkg.mod.Cls`, but `Cls` in
+                        // `__main__`), unlike the generic "has no
+                        // attribute" path which uses `tp_name`.
+                        None => {
+                            let ty = inst.cls().fully_qualified_name();
+                            let mut msg = String::with_capacity(ty.len() + slot.name.len() + 30);
+                            msg.push('\'');
+                            msg.push_str(&ty);
+                            msg.push_str("' object has no attribute '");
+                            msg.push_str(&slot.name);
+                            msg.push('\'');
+                            Err(attribute_error(msg))
+                        }
                     }
                 }
                 _ => Err(type_error("slot descriptor requires an instance")),
@@ -48042,8 +48065,15 @@ impl Interpreter {
         }
         frame.pc = pc as u32 + 1;
         let pending = self.core_pending_enter(sw, frame, pc);
+        // (Only a site cooling down after a failed load, an odd count, asks
+        // whether the attribute is certainly missing; see
+        // `specialized_load_attr`'s `Cooldown` arm.)
+        let failing = matches!(
+            frame.code.caches.get(pc as u32),
+            weavepy_compiler::InlineCache::Cooldown(n) if n & 1 == 1
+        );
         let r = match ins.op {
-            OpCode::LoadAttr => match self.load_attr_certain_miss(frame, ins.arg) {
+            OpCode::LoadAttr if failing => match self.load_attr_certain_miss(frame, ins.arg) {
                 Some(err) => Err(err),
                 None => self.load_attr_step(frame, pc as u32, ins.arg),
             },
@@ -48162,9 +48192,19 @@ impl Interpreter {
                                 specialize::record_hit(op_idx);
                                 return Ok(v);
                             }
+                            // Unset slot: the member descriptor raises its
+                            // `AttributeError`, as the default lookup's data
+                            // descriptor step would (the guard rules out a
+                            // `__getattr__` hook), and the site stays as it is
+                            // (CPython's `LOAD_ATTR_SLOT` deopts for this
+                            // execution only).
+                            let cls = inst.cls();
+                            if let Some(sd @ Object::SlotDescriptor(_)) = cls.lookup(name) {
+                                let owner = Object::Type(cls);
+                                let result = self.descriptor_get(&sd, &receiver, &owner);
+                                return self.load_attr_finish(&receiver, name, result);
+                            }
                         }
-                        // Unset slot: the slow path raises the faithful
-                        // AttributeError.
                     }
                 }
                 self.deopt_load_attr_taken(frame, cache_pc, name_idx, receiver)
@@ -49325,6 +49365,7 @@ impl Interpreter {
                 )));
             };
             *ty.qualname.borrow_mut() = Some(s.to_string());
+            ty.forget_display_names();
             return Ok(());
         }
         // The `type.__dict__` / `type.__mro__` getsets have no setter in
@@ -69083,6 +69124,39 @@ fn attr_certainly_missing(obj: &Object, name: &str) -> bool {
     default_lookup_misses(obj, name, false)
 }
 
+/// `PyObject_GetAttr`'s augmentation of a propagating `AttributeError`
+/// (`exc`) with `.name` and `.obj`, for one built natively, whose storage
+/// lays both fields out (see `error::attribute_error`): stored in place
+/// when neither is set. `true` once settled; `false` leaves the decision
+/// to the general path (any other storage, or attributes in a `__dict__`).
+fn attr_error_context_laid_out(exc: &PyInstance, obj: &Object, name: &str) -> bool {
+    if exc.attr_count() != 0 {
+        return false;
+    }
+    // (Noted before the slots are borrowed: the exception holds `obj`
+    // from here on, or already holds another one.)
+    exc.note_slot_store(obj);
+    let Ok(mut slots) = exc.slots.try_borrow_mut() else {
+        return false;
+    };
+    let Some((name_at, obj_at)) = slots.laid_out_pair_mut("name", "obj") else {
+        return false;
+    };
+    let unset = |v: &Object| matches!(v, Object::Unbound | Object::None);
+    if !(unset(name_at) && unset(obj_at)) {
+        return true;
+    }
+    // (A `str` needs no note.)
+    let name = Object::from_str(name);
+    let old = [
+        std::mem::replace(name_at, name),
+        std::mem::replace(obj_at, obj.clone()),
+    ];
+    drop(slots);
+    drop(old);
+    true
+}
+
 /// Whether the default `object.__getattribute__` lookup of `obj.name`
 /// certainly ends in its `AttributeError` without running any code (see
 /// [`attr_certainly_missing`]), ignoring a `__getattr__` hook on the class
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index f7fd5501..c6fe35b6 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -913,6 +913,10 @@ pub struct TypeObject {
     /// The leaf burst's per-class attribute-kind cache (see
     /// [`LeafAttrCache`]).
     pub leaf_attrs: LeafAttrCache,
+    /// [`Self::display_names`]'s memo: the version, dict stamp and C
+    /// `tp_name` it was computed under, the qualified name, and whether
+    /// the module is `__main__`.
+    display_name_memo: RefCell<Option<(u64, u64, usize, String, bool)>>,
     /// The most attributes an instance's `__dict__` was seen to hold
     /// (capped): new instances' dicts are presized to it, so `__init__`'s
     /// stores do not regrow them.
@@ -1378,6 +1382,7 @@ impl TypeObject {
             flags,
             metaclass: RefCell::new(None),
             leaf_attrs: LeafAttrCache::new(),
+            display_name_memo: RefCell::new(None),
             inst_dict_hint: std::sync::atomic::AtomicU32::new(0),
             shared_keys: crate::sync::LazyArc::new(),
             native_kind: Cell::new(0),
@@ -2283,13 +2288,60 @@ impl TypeObject {
         }
     }
 
+    /// CPython's `_PyType_GetFullyQualifiedName(type, '.')` (the `%T`
+    /// format): [`Self::qualified_display_name`], but without a
+    /// `__main__` prefix either.
+    pub fn fully_qualified_name(&self) -> String {
+        let (full, main) = self.display_names();
+        match full.strip_prefix("__main__.") {
+            Some(rest) if main => rest.to_owned(),
+            _ => full,
+        }
+    }
+
+    /// `module.qualname` as CPython's `type_repr` prints it (no module for
+    /// `builtins`).
     pub fn qualified_display_name(&self) -> String {
+        self.display_names().0
+    }
+
+    /// [`Self::qualified_display_name`], and whether the module is
+    /// `__main__`, memoized while the class's version, its dict's key
+    /// layout and its C `tp_name` stand (a `__qualname__` assignment
+    /// clears the memo).
+    fn display_names(&self) -> (String, bool) {
+        let ver = self.attr_version.get();
+        let stamp = self.dict.try_borrow().map_or(0, |d| d.mutation_stamp());
+        let tp_name = self.c_tp_name.get().map_or(0, |n| n.as_ptr() as usize);
+        if let Ok(memo) = self.display_name_memo.try_borrow() {
+            if let Some((v, s, t, name, main)) = &*memo {
+                if *v == ver && *s == stamp && *t == tp_name && stamp != 0 {
+                    return (name.clone(), *main);
+                }
+            }
+        }
+        let names = self.compute_display_names();
+        if let Ok(mut memo) = self.display_name_memo.try_borrow_mut() {
+            *memo = Some((ver, stamp, tp_name, names.0.clone(), names.1));
+        }
+        names
+    }
+
+    /// Forget [`Self::display_names`]'s memo (the class's qualified name
+    /// changed under an unchanged version).
+    pub(crate) fn forget_display_names(&self) {
+        if let Ok(mut memo) = self.display_name_memo.try_borrow_mut() {
+            *memo = None;
+        }
+    }
+
+    fn compute_display_names(&self) -> (String, bool) {
         let dict = self.dict.borrow();
         // Only honour *string* entries — some built-in types carry a
         // `__qualname__`/`__module__` *property descriptor* (for their
         // instances) in the dict, which must not leak into the class
         // repr (`type(gen)` printing `<class '<property object>'>`).
-        let as_str = |name: &'static str| match dict.get(&DictKey(Object::from_static(name))) {
+        let as_str = |name: &'static str| match dict.get(&crate::object::StrKey(name)) {
             Some(Object::Str(s)) => Some(s.as_ref().to_owned()),
             _ => None,
         };
@@ -2306,10 +2358,18 @@ impl TypeObject {
         let qual = as_str("__qualname__")
             .or_else(|| self.qualname.borrow().clone())
             .unwrap_or_else(|| self.name.clone());
-        match module.as_deref() {
+        let main = module.as_deref() == Some("__main__");
+        let full = match module.as_deref() {
             None | Some("builtins") | Some("") => qual,
-            Some(m) => format!("{m}.{qual}"),
-        }
+            Some(m) => {
+                let mut s = String::with_capacity(m.len() + 1 + qual.len());
+                s.push_str(m);
+                s.push('.');
+                s.push_str(&qual);
+                s
+            }
+        };
+        (full, main)
     }
 
     /// The name CPython's `tp_name`-based error text prints: the full
@@ -3341,6 +3401,36 @@ impl SlotStorage {
         }
     }
 
+    /// The laid-out places (holding `Unbound` while unset) of the two
+    /// distinct slots `a` and `b`, when the storage's fixed layout holds
+    /// both (an `AttributeError`'s `name` and `obj`). `None` otherwise.
+    pub(crate) fn laid_out_pair_mut(
+        &mut self,
+        a: &str,
+        b: &str,
+    ) -> Option<(&mut Object, &mut Object)> {
+        let SlotData::Fixed { layout, values } = &mut self.data else {
+            return None;
+        };
+        let pos = |name: &str| {
+            layout.iter().position(
+                |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)),
+            )
+        };
+        let (i, j) = (pos(a)?, pos(b)?);
+        if i == j || values.len() != layout.len() {
+            return None;
+        }
+        let (lo, hi) = (i.min(j), i.max(j));
+        let (head, tail) = values.split_at_mut(hi);
+        let (first, second) = (&mut head[lo], &mut tail[0]);
+        Some(if i < j {
+            (first, second)
+        } else {
+            (second, first)
+        })
+    }
+
     pub fn insert(&mut self, name: &str, value: Object) -> Option<Object> {
         self.insert_with_key(name, value, || slot_key(name))
     }
@@ -3560,6 +3650,15 @@ impl SlotStorage {
         self.get_mut(name)
     }
 
+    /// Never laid out here (see the 64-bit variant).
+    pub(crate) fn laid_out_pair_mut(
+        &mut self,
+        _a: &str,
+        _b: &str,
+    ) -> Option<(&mut Object, &mut Object)> {
+        None
+    }
+
     /// Never laid out here (see the 64-bit variant).
     #[inline]
     pub(crate) fn exc_tail_mut(&mut self, _which: ExcTail) -> Option<&mut Object> {
diff --git a/tests/regrtest/test_attribute_slow_arms.py b/tests/regrtest/test_attribute_slow_arms.py
index f8e9a9d6..858c48bb 100644
--- a/tests/regrtest/test_attribute_slow_arms.py
+++ b/tests/regrtest/test_attribute_slow_arms.py
@@ -180,6 +180,77 @@ class C:
         self.assertEqual(str(cm.exception), "'C' object has no attribute 'xy'")
 
 
+class Slotted:
+    __slots__ = ("_s", "_t")
+
+    def __init__(self):
+        self._t = 1
+
+    def get(self):
+        try:
+            return self._s
+        except AttributeError as e:
+            return e
+
+
+class SlotMissTest(unittest.TestCase):
+    def test_unset_slot_errors(self):
+        o = Slotted()
+        for _ in range(N):
+            e = o.get()
+        self.assertIsInstance(e, AttributeError)
+        self.assertEqual(e.name, "_s")
+        self.assertIs(e.obj, o)
+        # `%T`: the module, unless it is `builtins` or `__main__`.
+        mod = Slotted.__module__
+        prefix = "" if mod in ("builtins", "__main__") else mod + "."
+        self.assertEqual(str(e), f"'{prefix}Slotted' object has no attribute '_s'")
+
+    def test_site_serves_set_and_unset(self):
+        objs = [Slotted() for _ in range(4)]
+        for i, o in enumerate(objs):
+            if i % 2:
+                o._s = i
+        for _ in range(N):
+            got = [o.get() for o in objs]
+        self.assertIsInstance(got[0], AttributeError)
+        self.assertEqual(got[1], 1)
+        self.assertIsInstance(got[2], AttributeError)
+        self.assertEqual(got[3], 3)
+        objs[0]._s = "now"
+        del objs[1]._s
+        self.assertEqual(objs[0].get(), "now")
+        self.assertIsInstance(objs[1].get(), AttributeError)
+
+    def test_renamed_class(self):
+        class R:
+            __slots__ = ("_s",)
+
+        o = R()
+        for _ in range(N):
+            with self.assertRaises(AttributeError):
+                o._s
+        R.__qualname__ = "Renamed"
+        with self.assertRaises(AttributeError) as cm:
+            o._s
+        self.assertIn("Renamed' object has no attribute '_s'", str(cm.exception))
+        R.__module__ = "elsewhere"
+        with self.assertRaises(AttributeError) as cm:
+            o._s
+        self.assertEqual(str(cm.exception), "'elsewhere.Renamed' object has no attribute '_s'")
+
+    def test_getattr_added_later(self):
+        class H:
+            __slots__ = ("_s",)
+
+        o = H()
+        for _ in range(N):
+            with self.assertRaises(AttributeError):
+                o._s
+        H.__getattr__ = lambda self, name: "hook"
+        self.assertEqual(o._s, "hook")
+
+
 class Base:
     def __new__(cls, *args):
         return super().__new__(cls)

From a6bb2f1da2cfb9890dddfb8265f11adea4372125 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 21:01:58 -0700
Subject: [PATCH 139/173] perf: store class attributes and read property sites
 without the lanes

`Cls.attr = v` on a heap class under `type`'s default `__setattr__` is
stored by the core loop and the compiled code's store helper directly
(the plain end of `set_type_attr_direct`: a non-dunder name, no custom
metaclass, no type watcher, a replaced value that leaves by a plain
decrement), instead of leaving for the full STORE_ATTR handler. Bumping
a class's version no longer allocates its walk when it has no live
subclasses.

A compiled LOAD_ATTR at a site that caches a property getter calls the
getter first (CPython's LOAD_ATTR_PROPERTY), ahead of the cached reads
that would all decline for it. A `__getattr__` hook that is a plain
function is called without a bound method.

Measured (instructions per iteration, host, PYTHONHASHSEED=0):
  D.k = i (class attribute store)   2563 -> 501 (CPython 673)
  two property reads                 2331 -> 1191 (CPython 655)
  o.zz through __getattr__           5674 -> 5163 (CPython 1086)
Suite: callgrind totals for stdlib_utils and class_creation are within
0.2% of the previous commit (host geomean 1.414x -> 1.417x is noise
under load).
---
 crates/weavepy-vm/src/frame_jit.rs         |  78 ++++++++++--
 crates/weavepy-vm/src/lib.rs               |  76 ++++++++++++
 crates/weavepy-vm/src/types.rs             |  10 +-
 tests/regrtest/test_attribute_slow_arms.py | 131 +++++++++++++++++++++
 4 files changed, 281 insertions(+), 14 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index f267e823..2ac01bf2 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3995,6 +3995,11 @@ unsafe extern "C" fn h_stack_store_attr(code: *const CodeObject, pc: u64, at: *m
                 let arg = code.instructions[pc].arg;
                 return u32::from(!Interpreter::core_function_store(code, arg, at.add(1)));
             }
+            // A plain class's own attribute (`Cls.count = n`).
+            Object::Type(_) => {
+                let arg = code.instructions[pc].arg;
+                return u32::from(!Interpreter::core_class_store(code, arg, at.add(1)));
+            }
             _ => return 1,
         };
         if !droppable(recv)
@@ -4500,7 +4505,8 @@ unsafe extern "C" fn h_binop_dunder(
 /// ([`attr_lane`]); the result at stack slot `slot`. The receiver is the
 /// stack's at `slot` (`local` 0) or the local at `recv`, read in place.
 /// `0` done, `CALL_DECLINED` declined untouched, `RAISED` or `RELOAD` as
-/// [`h_call`]'s.
+/// [`h_call`]'s. With [`PROPERTY_ONLY`] in `local`, anything but the cached
+/// getter declines instead of taking the full handler.
 unsafe extern "C" fn h_property(
     st: *mut State,
     code: *const CodeObject,
@@ -4518,7 +4524,17 @@ unsafe extern "C" fn h_property(
         if st.sw.is_null() {
             return CALL_DECLINED;
         }
+        let only = local & PROPERTY_ONLY != 0;
+        let local = local & !PROPERTY_ONLY;
         let (pc, slot) = (pc as usize, slot as usize);
+        // The full handler, unless only the cached getter was asked for.
+        let lane = |st: &mut State| {
+            if only {
+                CALL_DECLINED
+            } else {
+                attr_lane(st, pc, slot, recv, local)
+            }
+        };
         // A frame's or code object's plain field, a live frame's line with
         // the running activation's own instruction.
         if matches!(&*recv, Object::Frame(_) | Object::Code(_)) && (local != 0 || droppable(&*recv))
@@ -4539,13 +4555,13 @@ unsafe extern "C" fn h_property(
             }
         }
         let Object::Instance(inst) = &*recv else {
-            return attr_lane(st, pc, slot, recv, local);
+            return lane(st);
         };
         let Some((getter, gcode)) = crate::code_vm_ext(&*code)
             .and_then(|ext| ext.method_slot(pc))
             .and_then(|ms| ms.getter(inst.cls_raw().attr_version.get()))
         else {
-            return attr_lane(st, pc, slot, recv, local);
+            return lane(st);
         };
         let at = st.stack.add(slot);
         // A pure leaf getter (no native body of its own: it never runs as
@@ -4580,13 +4596,17 @@ unsafe extern "C" fn h_property(
                     at.write(at.add(1).read());
                 }
                 drop(f);
-                attr_lane(st, pc, slot, recv, local)
+                lane(st)
             }
             Some(r) => r,
         }
     }
 }
 
+/// [`h_property`]'s flag (in `local`) for a call that wants the site's
+/// cached getter only.
+const PROPERTY_ONLY: u64 = 2;
+
 /// The `LOAD_ATTR` at `pc` (its receiver the stack's at `slot`, or the
 /// local at `recv` when `local`) as the full handler runs it
 /// (`Interpreter::core_attr_lane`), the frame synced and waiting on the
@@ -10661,11 +10681,48 @@ impl<'a> Lower<'a> {
                 self.b.ins().jump(miss, &[]);
             }
         }
-        // The core loop's cached reads.
         self.b.switch_to_block(miss);
+        let pcv = self.b.ins().iconst(types::I64, pc as i64);
+        // A property site: the getter the site cached for the receiver's
+        // class first, ahead of the cached reads that would all decline
+        // (CPython's `LOAD_ATTR_PROPERTY`).
+        let all_mem = self.vs.iter().all(|i| matches!(i, Item::Mem(_)));
+        let site_at = all_mem.then(|| {
+            let site: Box<DirectSite> = Box::default();
+            let at = std::ptr::from_ref::<DirectSite>(&site) as i64;
+            self.direct_sites.push(site);
+            at
+        });
+        if let (Some(site_at), true) = (site_at, crate::property_site(self.ext, pc)) {
+            let code = self.code_ptr();
+            let slotv = self.b.ins().iconst(types::I64, slot as i64);
+            let flags = PROPERTY_ONLY | u64::from(matches!(item, Item::Local(_)));
+            let local = self.b.ins().iconst(types::I64, flags as i64);
+            let site = self.b.ins().iconst(self.ptr, site_at);
+            let r = self
+                .call(
+                    h_property as *const () as usize,
+                    &[self.st, code, pcv, slotv, at, local, site],
+                    true,
+                )
+                .expect("returns");
+            let other = self.b.create_block();
+            let reads = self.b.create_block();
+            let leave = self.b.create_block();
+            self.b.ins().brif(r, other, &[], done, &[]);
+            self.b.switch_to_block(other);
+            let declined = self
+                .b
+                .ins()
+                .icmp_imm(IntCC::Equal, r, i64::from(CALL_DECLINED));
+            self.b.ins().brif(declined, reads, &[], leave, &[]);
+            self.b.switch_to_block(leave);
+            self.leave(r);
+            self.b.switch_to_block(reads);
+        }
+        // The core loop's cached reads.
         let code = self.code_ptr();
         let ext = self.ext_ptr();
-        let pcv = self.b.ins().iconst(types::I64, pc as i64);
         let c = self.b.ins().iconst(self.ptr, cache_at);
         let r = match item {
             Item::Local(_) => self.call(
@@ -10682,16 +10739,11 @@ impl<'a> Lower<'a> {
         .expect("returns");
         // A property whose getter the site has cached runs directly, when
         // nothing virtual lies below (its call can raise or switch).
-        if self.vs.iter().all(|i| matches!(i, Item::Mem(_))) {
+        if let Some(site_at) = site_at {
             let prop = self.b.create_block();
             self.b.ins().brif(r, prop, &[], done, &[]);
             self.b.switch_to_block(prop);
-            let site: Box<DirectSite> = Box::default();
-            let site_at = self
-                .b
-                .ins()
-                .iconst(self.ptr, std::ptr::from_ref::<DirectSite>(&site) as i64);
-            self.direct_sites.push(site);
+            let site_at = self.b.ins().iconst(self.ptr, site_at);
             let code = self.code_ptr();
             let slotv = self.b.ins().iconst(types::I64, slot as i64);
             let local = self
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 32527530..03d30dec 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -14876,6 +14876,21 @@ impl Interpreter {
                                 }
                                 offload!(OFFLOAD_HELPER);
                             }
+                            // A plain class's own attribute (`Cls.count = n`).
+                            Object::Type(_) => {
+                                // SAFETY: `len >= 2`: the receiver over the
+                                // value.
+                                if unsafe {
+                                    Self::core_class_store(code, ins.arg, base.add(len - 1))
+                                } {
+                                    len -= 2;
+                                    last = pc;
+                                    pc += 1;
+                                    after_release!();
+                                    continue;
+                                }
+                                offload!(OFFLOAD_HELPER);
+                            }
                             _ => offload!(OFFLOAD_HELPER),
                         };
                         if !Self::core_droppable(recv)
@@ -19309,6 +19324,63 @@ impl Interpreter {
         true
     }
 
+    /// `C.name = v` with the class at `recv` and the value below it: a
+    /// non-dunder name written to a heap class's own dict under `type`'s
+    /// default `__setattr__` (the plain end of [`Self::set_type_attr_direct`];
+    /// a custom metaclass or a type watcher keeps the full path), the
+    /// class's version advanced, both operands leaving the stack. `false`
+    /// touches nothing.
+    ///
+    /// # Safety
+    ///
+    /// `recv` is the core loop's top stack slot, the value the one below.
+    #[inline(never)]
+    unsafe fn core_class_store(code: &CodeObject, name_idx: u32, recv: *mut Object) -> bool {
+        // SAFETY: the caller's contract.
+        let (Object::Type(cls), value) = (unsafe { &*recv }, unsafe { &*recv.sub(1) }) else {
+            return false;
+        };
+        if crate::capi_watchers::types_active()
+            || !Self::core_drops_plain()
+            || !Self::core_droppable(unsafe { &*recv })
+            || cls.flags.is_builtin
+            || cls.immutable.get()
+            || cls.c_ext_ptr.get() != 0
+            || !Self::plain_metaclass(cls)
+        {
+            return false;
+        }
+        let Some(name @ Object::Str(n)) = code_name_obj(code, name_idx) else {
+            return false;
+        };
+        if n.starts_with("__") {
+            return false;
+        }
+        let old = {
+            let Ok(mut d) = cls.dict.try_borrow_mut() else {
+                return false;
+            };
+            match d.get_mut(&crate::object::StrKey(n)) {
+                // (The value it replaces leaves by a plain decrement.)
+                Some(slot) if !Self::core_droppable(slot) => return false,
+                // SAFETY: the value moves into the dictionary; its slot is
+                // forgotten below.
+                Some(slot) => Some(std::mem::replace(slot, unsafe { std::ptr::read(value) })),
+                None => {
+                    // SAFETY: as above.
+                    d.insert(DictKey(name.clone()), unsafe { std::ptr::read(value) });
+                    None
+                }
+            }
+        };
+        cls.bump_attr_version();
+        // SAFETY: the class leaves by a plain decrement (checked droppable);
+        // the value's slot moved out above.
+        unsafe { drop_hot(recv.read()) };
+        drop(old);
+        true
+    }
+
     /// Whether a release reports to nothing (no dict or function watcher,
     /// no reference tracing), so a droppable value leaves by a plain
     /// decrement (see [`Self::core_droppable`]).
@@ -36070,6 +36142,10 @@ impl Interpreter {
         getattr: Object,
         name: &str,
     ) -> Result<Object, RuntimeError> {
+        // A plain function binds the instance as its first argument.
+        if let Object::Function(f) = &getattr {
+            return self.call_python(f, &[instance_obj.clone(), Object::from_str(name)], &[]);
+        }
         // `dispatch` so a descriptor `__getattr__` also honours its
         // `__get__` (test_descr test_getattr_hooks).
         let bound = Object::BoundMethod(Rc::new(BoundMethod::dispatch(
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index c6fe35b6..a2732e19 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -1860,9 +1860,17 @@ impl TypeObject {
     /// LOAD_ATTR/STORE_ATTR sites guard on the version and deopt.
     pub fn bump_attr_version(&self) {
         use std::sync::atomic::Ordering::Relaxed;
-        let mut visited: Vec<*const TypeObject> = vec![std::ptr::from_ref::<TypeObject>(self)];
         self.attr_version.bump();
         self.mro_kind.store(0, Relaxed);
+        // (A leaf class, the common case, has nothing to walk.)
+        if self
+            .subclasses
+            .try_borrow()
+            .is_ok_and(|subs| subs.iter().all(|w| w.strong_count() == 0))
+        {
+            return;
+        }
+        let mut visited: Vec<*const TypeObject> = vec![std::ptr::from_ref::<TypeObject>(self)];
         let mut queue: Vec<Rc<TypeObject>> = self.subclasses();
         while let Some(t) = queue.pop() {
             let ptr = Rc::as_ptr(&t);
diff --git a/tests/regrtest/test_attribute_slow_arms.py b/tests/regrtest/test_attribute_slow_arms.py
index 858c48bb..5d253769 100644
--- a/tests/regrtest/test_attribute_slow_arms.py
+++ b/tests/regrtest/test_attribute_slow_arms.py
@@ -251,6 +251,137 @@ class H:
         self.assertEqual(o._s, "hook")
 
 
+class ClassStoreTest(unittest.TestCase):
+    def test_store_is_seen(self):
+        class K:
+            n = 0
+
+        class Sub(K):
+            pass
+
+        k, s = K(), Sub()
+        for i in range(N):
+            K.n = i
+            self.assertEqual(k.n, i)
+            self.assertEqual(s.n, i)
+            self.assertEqual(Sub.n, i)
+        K.fresh = "new"
+        self.assertEqual(k.fresh, "new")
+
+    def test_metaclass_hooks_run(self):
+        seen = []
+
+        class Meta(type):
+            def __setattr__(cls, name, value):
+                seen.append(name)
+                super().__setattr__(name, value)
+
+        class M(metaclass=Meta):
+            n = 0
+
+        def store(c, v):
+            c.n = v
+
+        class P:
+            n = 0
+
+        for i in range(N):
+            store(P, i)
+        store(M, 5)
+        self.assertEqual(seen, ["n"])
+        self.assertEqual(M.n, 5)
+
+    def test_immutable_type(self):
+        def store(c):
+            c.attr = 1
+
+        class P:
+            pass
+
+        for _ in range(N):
+            store(P)
+        with self.assertRaises(TypeError):
+            store(int)
+
+    def test_replaced_value_finalized(self):
+        log = []
+
+        class Fin:
+            def __del__(self):
+                log.append("del")
+
+        class P:
+            v = None
+
+        def store(v):
+            P.v = v
+
+        for _ in range(N):
+            store(1)
+        store(Fin())
+        store(2)
+        self.assertEqual(log, ["del"])
+
+
+class PropertySiteTest(unittest.TestCase):
+    def test_polymorphic_site(self):
+        class A:
+            @property
+            def v(self):
+                return "prop"
+
+        class B:
+            def __init__(self):
+                self.v = "field"
+
+        def read(o):
+            return o.v
+
+        for _ in range(N):
+            self.assertEqual(read(A()), "prop")
+        self.assertEqual(read(B()), "field")
+        self.assertEqual(read(A()), "prop")
+
+    def test_getter_raises(self):
+        class A:
+            def __init__(self):
+                self.fail = False
+
+            @property
+            def v(self):
+                if self.fail:
+                    raise ValueError("boom")
+                return 1
+
+        a = A()
+
+        def read(o):
+            return o.v
+
+        for _ in range(N):
+            self.assertEqual(read(a), 1)
+        a.fail = True
+        with self.assertRaises(ValueError):
+            read(a)
+
+    def test_property_replaced(self):
+        class A:
+            @property
+            def v(self):
+                return 1
+
+        def read(o):
+            return o.v
+
+        a = A()
+        for _ in range(N):
+            self.assertEqual(read(a), 1)
+        A.v = property(lambda self: 2)
+        self.assertEqual(read(a), 2)
+        A.v = 3
+        self.assertEqual(read(a), 3)
+
+
 class Base:
     def __new__(cls, *args):
         return super().__new__(cls)

From 87067d6f8d97c2c2eac2819d2fe0a3fa1906cb8b Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 22:05:19 -0700
Subject: [PATCH 140/173] perf: cheaper class lookups for names, stores and
 comparison dunders

The type cache compared a name shorter than eight bytes byte by byte,
twice per hit (the cache entry, then the dict key it points at), and
hashed it through a stack copy. Both now read the name with two
overlapping loads.

An instance store at a site without a usable cache settles the class's
part of the plain-store question (default attribute access and setting,
no data descriptor on the name) once per class version instead of
resolving the name on the class every time.

A rich comparison that gives a subclass operand's reflected dunder
priority, or that runs the forward dunder after it, calls a plain
Python function on the class directly from the class's cache, as the
common forward path already did, instead of resolving and binding it.

Callgrind totals (per call) against the previous commit: oo_patterns
1427.9M -> 1395.4M (-2.3%), class_creation 536.3M -> 534.8M (-0.3%);
context_managers and sorting unchanged.
---
 crates/weavepy-vm/src/lib.rs               | 99 ++++++++++++++++------
 crates/weavepy-vm/src/types.rs             | 35 +++++---
 tests/regrtest/test_attribute_slow_arms.py | 48 +++++++++++
 3 files changed, 147 insertions(+), 35 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 03d30dec..78aaa078 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -29109,22 +29109,27 @@ impl Interpreter {
                     // establishes, read from the class's own caches).
                     _ => {
                         let cls = inst.cls_raw();
-                        if !Self::default_getattribute(cls)
-                            || cls.flags.is_builtin
-                            || !cls.setattr_is_default()
-                            || cls.forbids_dict
-                            || crate::object::exotic_str_keys_possible()
-                            || (name.starts_with("__") && name != "__doc__")
-                            // An exception's `args` is a `BaseException`
-                            // descriptor.
-                            || (name == "args"
-                                && cls.is_subclass_of(&builtin_types().base_exception))
+                        if crate::object::exotic_str_keys_possible()
                             || inst.c_body.get() != 0
-                            || !matches!(
-                                cls.lookup(name),
-                                None | Some(Object::Function(_) | Object::Str(_) | Object::None)
-                            )
                             || crate::capi_watchers::dicts_active()
+                            // (The class's part, settled per class version.)
+                            || !cls.memo_verdict(crate::types::Verdict::PlainStore, name, || {
+                                Self::default_getattribute(cls)
+                                    && !cls.flags.is_builtin
+                                    && cls.setattr_is_default()
+                                    && !cls.forbids_dict
+                                    && !(name.starts_with("__") && name != "__doc__")
+                                    // An exception's `args` is a
+                                    // `BaseException` descriptor.
+                                    && !(name == "args"
+                                        && cls.is_subclass_of(&builtin_types().base_exception))
+                                    && matches!(
+                                        cls.lookup(name),
+                                        None | Some(
+                                            Object::Function(_) | Object::Str(_) | Object::None
+                                        )
+                                    )
+                            })
                         {
                             return None;
                         }
@@ -46698,6 +46703,23 @@ impl Interpreter {
         }
     }
 
+    /// `type(x).<op's dunder>` for an instance `x` when it is a plain
+    /// Python function, from the class's cache (see
+    /// [`Self::instance_dunder_function`]): called with `x` first, it is
+    /// what [`Self::cmp_method`] binds.
+    fn cmp_function(x: &Object, op: CompareKind) -> Option<Rc<PyFunction>> {
+        /// Class cache keys per comparison dunder.
+        static CMP_KEYS: [u8; 8] = [0; 8];
+        let Object::Instance(inst) = x else {
+            return None;
+        };
+        Self::instance_dunder_function(
+            inst,
+            std::ptr::addr_of!(CMP_KEYS[op as usize]) as usize,
+            cmp_dunder(op).0,
+        )
+    }
+
     /// Bool-valued rich comparison — the truthiness of [`Self::rich_compare_obj`].
     /// Used everywhere a comparison feeds control flow or a native algorithm
     /// (`min`/`max`, `in`, sort merges, the `==` sentinel probes).
@@ -46835,14 +46857,7 @@ impl Interpreter {
             if std::ptr::eq(ia.cls_raw(), ib.cls_raw())
                 || !ib.cls_raw().is_subclass_of(ia.cls_raw())
             {
-                /// Class cache keys per comparison (see
-                /// `instance_dunder_function`).
-                static CMP_KEYS: [u8; 8] = [0; 8];
-                if let Some(f) = Self::instance_dunder_function(
-                    ia,
-                    std::ptr::addr_of!(CMP_KEYS[op as usize]) as usize,
-                    cmp_dunder(op).0,
-                ) {
+                if let Some(f) = Self::cmp_function(a, op) {
                     let r = self.call_dunder_function(f, &[a.clone(), b.clone()])?;
                     if !r.is_same(&crate::vm_singletons::not_implemented()) {
                         return Ok(r);
@@ -46961,9 +46976,12 @@ impl Interpreter {
         // interface's richer `__eq__` first (test_ipaddress), and
         // `addr < intf` must run the interface's total_ordering
         // `__gt__` first.
+        // (A plain function on an instance's class, from the class's
+        // cache, is called directly below, as `cmp_method` would bind it.)
+        let b_function = Self::cmp_function(b, cmp_swapped(op));
         let reflected_first = !Rc::ptr_eq(&cls_b, &cls_a)
             && cls_b.is_subclass_of(&cls_a)
-            && (cls_b.lookup(swapped).is_some() || b_foreign);
+            && (b_function.is_some() || b_foreign || cls_b.lookup(swapped).is_some());
         let mut tried_reflected = false;
         // Track whether each operand's *own* `__ne__` slot actually ran.
         // `object.__ne__` (installed on every class) already derives from the
@@ -46983,7 +47001,13 @@ impl Interpreter {
                     Err(e) => return Err(e),
                 }
             }
-            if let Some(method) = self.cmp_method(b, swapped, globals) {
+            if let Some(f) = b_function.clone() {
+                b_slot_ran = true;
+                let r = self.call_dunder_function(f, &[b.clone(), a.clone()])?;
+                if !r.is_same(&not_impl) {
+                    return Ok(r);
+                }
+            } else if let Some(method) = self.cmp_method(b, swapped, globals) {
                 b_slot_ran = true;
                 let r = self.call(&method, std::slice::from_ref(a), &[], globals)?;
                 if !r.is_same(&not_impl) {
@@ -47002,6 +47026,12 @@ impl Interpreter {
         }
         if forward_ran {
             a_slot_ran = true;
+        } else if let Some(f) = Self::cmp_function(a, op) {
+            a_slot_ran = true;
+            let r = self.call_dunder_function(f, &[a.clone(), b.clone()])?;
+            if !r.is_same(&not_impl) {
+                return Ok(r);
+            }
         } else if let Some(method) = self.cmp_method(a, dunder, globals) {
             a_slot_ran = true;
             let r = self.call(&method, std::slice::from_ref(b), &[], globals)?;
@@ -47019,7 +47049,13 @@ impl Interpreter {
                     Err(e) => return Err(e),
                 }
             }
-            if let Some(method) = self.cmp_method(b, swapped, globals) {
+            if let Some(f) = b_function {
+                b_slot_ran = true;
+                let r = self.call_dunder_function(f, &[b.clone(), a.clone()])?;
+                if !r.is_same(&not_impl) {
+                    return Ok(r);
+                }
+            } else if let Some(method) = self.cmp_method(b, swapped, globals) {
                 b_slot_ran = true;
                 let r = self.call(&method, std::slice::from_ref(a), &[], globals)?;
                 if !r.is_same(&not_impl) {
@@ -73440,6 +73476,19 @@ fn binop_dunders(op: BinOpKind) -> (&'static str, &'static str) {
     }
 }
 
+/// The comparison whose forward dunder is `op`'s reflected one.
+fn cmp_swapped(op: CompareKind) -> CompareKind {
+    use CompareKind as C;
+    match op {
+        C::Eq => C::Eq,
+        C::NotEq => C::NotEq,
+        C::Lt => C::Gt,
+        C::LtE => C::GtE,
+        C::Gt => C::Lt,
+        C::GtE => C::LtE,
+    }
+}
+
 fn cmp_dunder(op: CompareKind) -> (&'static str, &'static str) {
     use CompareKind as C;
     match op {
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index a2732e19..94a080c4 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -162,6 +162,10 @@ pub(crate) enum Verdict {
     /// No builtin base on the class's MRO supplies a slot wrapper or a
     /// type-dict entry `name` (see `builtin_slot_wrapper`).
     NoSlotWrapper = 4,
+    /// An instance store of `name` is a plain instance-dict store: default
+    /// attribute access and setting, and no class attribute in the way
+    /// but a function or a plain value (see `Interpreter::leaf_store_attr`).
+    PlainStore = 5,
 }
 
 pub struct AttrVersion(std::sync::atomic::AtomicU64);
@@ -337,14 +341,31 @@ pub(crate) mod type_cache {
                 u64::from_le_bytes(b[n - 8..].try_into().expect("8 bytes")),
             )
         } else {
-            let mut w = [0u8; 8];
-            w[..n].copy_from_slice(b);
-            (u64::from_le_bytes(w), 0)
+            (short_word(b), 0)
         };
         let x = (head ^ tail.rotate_left(29) ^ (n as u64)).wrapping_mul(0x9E37_79B9_7F4A_7C15);
         x ^ (x >> 29)
     }
 
+    /// The bytes of a name shorter than eight bytes as one word, from two
+    /// overlapping loads that together cover it (so two names of the same
+    /// length are equal exactly when their words are).
+    #[inline(always)]
+    fn short_word(b: &[u8]) -> u64 {
+        let n = b.len();
+        if n >= 4 {
+            let lo = u32::from_le_bytes(b[..4].try_into().expect("4 bytes"));
+            let hi = u32::from_le_bytes(b[n - 4..].try_into().expect("4 bytes"));
+            u64::from(lo) | u64::from(hi) << 32
+        } else if n >= 2 {
+            let lo = u16::from_le_bytes(b[..2].try_into().expect("2 bytes"));
+            let hi = u16::from_le_bytes(b[n - 2..].try_into().expect("2 bytes"));
+            u64::from(lo) | u64::from(hi) << 16
+        } else {
+            b.first().map_or(0, |&c| u64::from(c))
+        }
+    }
+
     /// `a == b` for attribute names, in line for the short ones (most
     /// dunders and identifiers fit in sixteen bytes).
     #[inline]
@@ -361,13 +382,7 @@ pub(crate) mod type_cache {
         if n >= 8 {
             return w(a, 0) == w(b, 0) && w(a, n - 8) == w(b, n - 8);
         }
-        let mut x = 0u64;
-        let mut y = 0u64;
-        for i in 0..n {
-            x |= u64::from(a[i]) << (8 * i);
-            y |= u64::from(b[i]) << (8 * i);
-        }
-        x == y
+        short_word(a) == short_word(b)
     }
 
     #[inline]
diff --git a/tests/regrtest/test_attribute_slow_arms.py b/tests/regrtest/test_attribute_slow_arms.py
index 5d253769..2a5ce0e4 100644
--- a/tests/regrtest/test_attribute_slow_arms.py
+++ b/tests/regrtest/test_attribute_slow_arms.py
@@ -382,6 +382,54 @@ def read(o):
         self.assertEqual(read(a), 3)
 
 
+class ReflectedCompareTest(unittest.TestCase):
+    def test_subclass_reflected_first(self):
+        log = []
+
+        class A:
+            def __lt__(self, other):
+                log.append("A.lt")
+                return "A"
+
+            def __eq__(self, other):
+                log.append("A.eq")
+                return False
+
+        class B(A):
+            def __gt__(self, other):
+                log.append("B.gt")
+                return "B"
+
+            def __eq__(self, other):
+                log.append("B.eq")
+                return NotImplemented
+
+        a, b = A(), B()
+        for _ in range(N):
+            log.clear()
+            self.assertEqual(a < b, "B")
+            self.assertEqual(log, ["B.gt"])
+        log.clear()
+        self.assertFalse(a == b)
+        self.assertEqual(log, ["B.eq", "A.eq"])
+        B.__gt__ = lambda self, other: NotImplemented
+        log.clear()
+        self.assertEqual(a < b, "A")
+        self.assertEqual(log, ["A.lt"])
+
+    def test_unorderable(self):
+        class A:
+            pass
+
+        class B(A):
+            def __gt__(self, other):
+                return NotImplemented
+
+        for _ in range(N):
+            with self.assertRaises(TypeError):
+                A() < B()
+
+
 class Base:
     def __new__(cls, *args):
         return super().__new__(cls)

From 71036ee423561a34044f6bcdfac3a5fae62eac08 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 18 Oct 2026 21:24:15 -0700
Subject: [PATCH 141/173] perf: read inst.__class__ and local functions'
 attributes without the full lookup

`inst.__class__` on a heap class whose MRO overrides neither `__class__`
nor `__getattribute__` is `object`'s descriptor answering the class, so
it is answered before the default lookup's passes (`_abc_instancecheck`
reads it for every isinstance against an ABC, at about 1.3k
instructions a read).

Compiled code's local-receiver attribute helper reads a function's
stored attribute (`wrapper.calls`) as the stack-receiver helper and the
core loop's arm do, instead of leaving for the full handler.

The generic LOAD_ATTR and STORE_ATTR arms borrow the name from the code
object instead of copying it into a fresh String.

Callgrind totals (per call), measured before rebasing onto the merged
calls and closures work: oo_patterns 1395.7M -> 1373.1M (-1.6%),
closures_decorators 1709.3M -> 1668.1M (-2.4%).
---
 crates/weavepy-vm/src/frame_jit.rs         |  1 +
 crates/weavepy-vm/src/lib.rs               | 57 +++++++++++------
 tests/regrtest/test_attribute_slow_arms.py | 73 ++++++++++++++++++++++
 3 files changed, 113 insertions(+), 18 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 2ac01bf2..d42155a5 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3950,6 +3950,7 @@ unsafe extern "C" fn h_local_attr(
             }
             Object::Instance(_) => Interpreter::core_local_attr(Some(ext), code, recv, pc, arg),
             Object::Type(cls) => type_attr(code, ext, cls, pc, arg),
+            Object::Function(f) => Interpreter::leaf_function_read(code, f, arg),
             // A frame's or code object's plain field (a live frame's line
             // needs the full lane, which makes its `lasti` current).
             Object::Code(_) | Object::Frame(_) => {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 78aaa078..5f468959 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -33910,6 +33910,15 @@ impl Interpreter {
 
     // ---------- helpers ----------
 
+    /// [`Self::name_at`] without the copy, for a caller that keeps the
+    /// code alive across the use.
+    fn name_ref(code: &CodeObject, arg: u32) -> Result<&str, RuntimeError> {
+        code.names
+            .get(arg as usize)
+            .map(|n| n.as_str())
+            .ok_or_else(|| RuntimeError::Internal("bad name index".to_owned()))
+    }
+
     fn name_at(&self, code: &CodeObject, arg: u32) -> Result<String, RuntimeError> {
         code.names
             .get(arg as usize)
@@ -36099,6 +36108,17 @@ impl Interpreter {
                 Rc::strong_count(inst),
             );
         }
+        // `inst.__class__` (`_abc_instancecheck`, `isinstance` misses,
+        // `copy`): `object`'s descriptor answering the class, when nothing
+        // on the MRO overrides `__class__` or `__getattribute__` (a
+        // builtin class's instances, weak proxies among them, keep the
+        // full lookup).
+        if name == "__class__" && inst.c_body.get() == 0 {
+            let cls = inst.cls();
+            if !cls.flags.is_builtin && cls.class_attr_is_plain() {
+                return Ok(Object::Type(cls));
+            }
+        }
         let result = if let Some(getattribute) = self.user_getattribute(&inst.cls()) {
             // `dispatch` (not `new`): a `__getattribute__` that is itself a
             // descriptor (`__getattribute__ = SomeDescriptor()`) must have
@@ -48435,9 +48455,9 @@ impl Interpreter {
                 self.deopt_load_attr_taken(frame, cache_pc, name_idx, receiver)
             }
             IC::Empty => {
-                let name = self.name_at(&frame.code, name_idx)?;
+                let name = Self::name_ref(&frame.code, name_idx)?;
                 let obj = receiver;
-                let result = self.load_attr(&obj, &name);
+                let result = self.load_attr(&obj, name);
                 // Specialize *after* the generic lookup. The decision's
                 // dict-index probe hashes `name` into the instance dict and
                 // can run a hostile key's `__eq__` (test_descr
@@ -48449,7 +48469,7 @@ impl Interpreter {
                 // when its load failed, an even one otherwise (see the
                 // `Cooldown` arm).
                 let decision = if result.is_ok() {
-                    match specialize::attempt_specialize_load_attr(&obj, &name) {
+                    match specialize::attempt_specialize_load_attr(&obj, name) {
                         IC::Cooldown(n) => IC::Cooldown(n.saturating_mul(2) & !1),
                         decision => decision,
                     }
@@ -48482,12 +48502,12 @@ impl Interpreter {
                         return Err(err);
                     }
                 }
-                let name = self.name_at(&frame.code, name_idx)?;
-                self.load_attr(&receiver, &name)
+                let name = Self::name_ref(&frame.code, name_idx)?;
+                self.load_attr(&receiver, name)
             }
             _ => {
-                let name = self.name_at(&frame.code, name_idx)?;
-                self.load_attr(&receiver, &name)
+                let name = Self::name_ref(&frame.code, name_idx)?;
+                self.load_attr(&receiver, name)
             }
         }
     }
@@ -48509,8 +48529,8 @@ impl Interpreter {
             .code
             .caches
             .set(cache_pc, weavepy_compiler::InlineCache::Cooldown(2 * COOLDOWN));
-        let name = self.name_at(&frame.code, name_idx)?;
-        self.load_attr(&receiver, &name)
+        let name = Self::name_ref(&frame.code, name_idx)?;
+        self.load_attr(&receiver, name)
     }
 
     /// Specialized `STORE_ATTR`. Stack discipline matches the
@@ -48768,9 +48788,10 @@ impl Interpreter {
                 self.deopt_store_attr_taken(frame, cache_pc, name_idx, receiver)
             }
             IC::Empty => {
-                let name = self.name_at(&frame.code, name_idx)?;
+                let code = frame.code.clone();
+                let name = Self::name_ref(&code, name_idx)?;
                 specialize::record_specialize_attempt(op_idx);
-                let decision = specialize::attempt_specialize_store_attr(&receiver, &name);
+                let decision = specialize::attempt_specialize_store_attr(&receiver, name);
                 frame.code.caches.set(cache_pc, decision);
                 if matches!(decision, IC::Cooldown(_)) {
                     specialize::record_specialize_skip(op_idx);
@@ -48778,7 +48799,7 @@ impl Interpreter {
                     specialize::record_specialize_success(op_idx);
                 }
                 let val = frame.pop()?;
-                self.store_attr(&receiver, &name, val)
+                self.store_attr(&receiver, name, val)
             }
             IC::Cooldown(n) => {
                 let next = if n > 0 {
@@ -48788,13 +48809,13 @@ impl Interpreter {
                 };
                 frame.code.caches.set(cache_pc, next);
                 let val = frame.pop()?;
-                let name = self.name_at(&frame.code, name_idx)?;
-                self.store_attr(&receiver, &name, val)
+                let name = Self::name_ref(&frame.code, name_idx)?;
+                self.store_attr(&receiver, name, val)
             }
             _ => {
                 let val = frame.pop()?;
-                let name = self.name_at(&frame.code, name_idx)?;
-                self.store_attr(&receiver, &name, val)
+                let name = Self::name_ref(&frame.code, name_idx)?;
+                self.store_attr(&receiver, name, val)
             }
         }
     }
@@ -48815,8 +48836,8 @@ impl Interpreter {
             .caches
             .set(cache_pc, weavepy_compiler::InlineCache::Cooldown(COOLDOWN));
         let val = frame.pop()?;
-        let name = self.name_at(&frame.code, name_idx)?;
-        self.store_attr(&receiver, &name, val)
+        let name = Self::name_ref(&frame.code, name_idx)?;
+        self.store_attr(&receiver, name, val)
     }
 
     /// Specialized `FOR_ITER`. Returns `Ok(true)` when the fast
diff --git a/tests/regrtest/test_attribute_slow_arms.py b/tests/regrtest/test_attribute_slow_arms.py
index 2a5ce0e4..d3233259 100644
--- a/tests/regrtest/test_attribute_slow_arms.py
+++ b/tests/regrtest/test_attribute_slow_arms.py
@@ -430,6 +430,79 @@ def __gt__(self, other):
                 A() < B()
 
 
+class FunctionAttrTest(unittest.TestCase):
+    def test_counter_on_wrapper(self):
+        def counted(fn):
+            def wrapper(*args):
+                wrapper.calls += 1
+                return fn(*args)
+
+            wrapper.calls = 0
+            return wrapper
+
+        @counted
+        def double(x):
+            return 2 * x
+
+        for i in range(N):
+            self.assertEqual(double(i), 2 * i)
+        self.assertEqual(double.calls, N)
+        double.__dict__ = {"calls": 100}
+        double(1)
+        self.assertEqual(double.calls, 101)
+        del double.calls
+        with self.assertRaises(AttributeError):
+            double(1)
+
+    def test_function_dunders(self):
+        def f():
+            pass
+
+        def read(fn):
+            return fn.__name__, fn.__qualname__
+
+        for _ in range(N):
+            self.assertEqual(read(f)[0], "f")
+        f.__name__ = "g"
+        self.assertEqual(read(f)[0], "g")
+
+
+class DunderClassTest(unittest.TestCase):
+    def test_class_reads(self):
+        import abc
+        import weakref
+
+        class A(abc.ABC):
+            pass
+
+        class B(A):
+            pass
+
+        class C:
+            pass
+
+        class D(A):
+            pass
+
+        class Lying:
+            @property
+            def __class__(self):
+                return int
+
+        b = B()
+        for _ in range(N):
+            self.assertIsInstance(b, A)
+            self.assertNotIsInstance(C(), A)
+            self.assertIs(b.__class__, B)
+        b.__class__ = D
+        self.assertIs(b.__class__, D)
+        self.assertNotIsInstance(b, B)
+        self.assertIs(Lying().__class__, int)
+        self.assertIsInstance(Lying(), int)
+        c = C()
+        self.assertIs(weakref.proxy(c).__class__, C)
+
+
 class Base:
     def __new__(cls, *args):
         return super().__new__(cls)

From 1f29e7890542021e03c4cece969a719e56ab4c2c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 00:02:40 -0700
Subject: [PATCH 142/173] perf: settle hasattr on builtin classes per class
 version

`hasattr(type(coro), '_asyncio_future_blocking')` (asyncio's `isfuture`,
for every coroutine it is handed) answered a repeated miss on a builtin
class through a thread-local map keyed by the class address, after
checking the class against the struct-sequence registry by hashing its
name, and copied the name argument into a fresh String. The miss is now
recorded in the type cache under the class's version (a struct-sequence
class, whose attributes can be set, moves its version when they are),
and `getattr`/`hasattr` borrow a plain `str` name.

The `__dict__` site check runs after a polymorphic site's own entries,
and only names `__dict__` take its out-of-line half.

Callgrind totals (per call) against the previous commit: async_tree
1301.4M -> 1296.5M (-0.4%), mini_interpreter 1874.7M -> 1872.1M,
coroutines and logging_mix unchanged.
---
 crates/weavepy-vm/src/lib.rs               | 103 +++++++--------------
 crates/weavepy-vm/src/types.rs             |  36 +++++++
 tests/regrtest/test_attribute_slow_arms.py |  25 +++++
 3 files changed, 93 insertions(+), 71 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 5f468959..47520787 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -28339,9 +28339,6 @@ impl Interpreter {
                 return Some(v);
             }
         }
-        if let Some(v) = inst_dict_attr(code, inst, cache_pc as usize, name_idx) {
-            return Some(v);
-        }
         let poly = code_attr_poly(code, cache_pc);
         let ver = inst.cls_raw().attr_version.get();
         // A never-specialized site resolves once and takes the indexed
@@ -28352,6 +28349,9 @@ impl Interpreter {
                 return Some(v);
             }
         }
+        if let Some(v) = inst_dict_attr(code, inst, cache_pc as usize, name_idx) {
+            return Some(v);
+        }
         match Self::leaf_resolve_instance_attr_ix(code, inst, name_idx)? {
             (LeafAttr::Value(v), ix) => {
                 if let Some(ix) = ix {
@@ -41106,7 +41106,7 @@ impl Interpreter {
         args: &[Object],
         _globals: &Rc<RefCell<DictData>>,
     ) -> Result<Object, RuntimeError> {
-        let name = match attr_name_arg(args.get(1)) {
+        let name = match attr_name_cow(args.get(1)) {
             Some(n) => n,
             None => {
                 return Err(type_error(format!(
@@ -41147,7 +41147,7 @@ impl Interpreter {
         args: &[Object],
         _globals: &Rc<RefCell<DictData>>,
     ) -> Result<Object, RuntimeError> {
-        let name = match attr_name_arg(args.get(1)) {
+        let name = match attr_name_cow(args.get(1)) {
             Some(n) => n,
             None => {
                 return Err(type_error(format!(
@@ -41164,22 +41164,23 @@ impl Interpreter {
         if attr_certainly_missing(&args[0], &name) {
             return Ok(Object::Bool(false));
         }
-        let immutable_type = match &args[0] {
-            Object::Type(t) if t.flags.is_builtin && !crate::stdlib::os::is_struct_seq_type(t) => {
-                Some(t)
-            }
+        // A builtin class's miss, remembered per class version (a
+        // struct-sequence class, whose attributes can be set, moves its
+        // version when they are).
+        let builtin_type = match &args[0] {
+            Object::Type(t) if t.flags.is_builtin => Some(t),
             _ => None,
         };
-        if let Some(t) = immutable_type {
-            if builtin_type_misses::contains(t, &name) {
+        if let Some(t) = builtin_type {
+            if t.verdict_cached(crate::types::Verdict::TypeAttrMissing, &name) == Some(true) {
                 return Ok(Object::Bool(false));
             }
         }
         match self.load_attr(&args[0], &name) {
             Ok(_) => Ok(Object::Bool(true)),
             Err(e) if self.is_attribute_error(&e) => {
-                if let Some(t) = immutable_type {
-                    builtin_type_misses::insert(t, &name);
+                if let Some(t) = builtin_type {
+                    t.verdict_record(crate::types::Verdict::TypeAttrMissing, &name, true);
                 }
                 Ok(Object::Bool(false))
             }
@@ -69197,57 +69198,6 @@ thread_local! {
     > = std::cell::RefCell::new(crate::fasthash::FxHashMap::default());
 }
 
-/// Attribute names a built-in type was found to lack. Built-in types are
-/// immutable, so a miss is permanent, and `hasattr` can answer a repeat
-/// without raising (asyncio's `isfuture` asks
-/// `hasattr(type(coro), '_asyncio_future_blocking')` for every coroutine
-/// it's handed). Their attributes resolve dynamically, so a first miss
-/// must still go through the full lookup. Entries are keyed by address
-/// and hold the type weakly, which keeps the address from being reused.
-mod builtin_type_misses {
-    use std::cell::RefCell;
-
-    use crate::fasthash::{FxHashMap, FxHashSet};
-    use crate::sync::{Rc, Weak};
-    use crate::types::TypeObject;
-
-    /// Past this many names the record starts over (a bound, not a cache
-    /// policy: real programs probe a handful of names).
-    const CAP: usize = 512;
-
-    type Misses = FxHashMap<usize, (Weak<TypeObject>, FxHashSet<Box<str>>)>;
-
-    thread_local! {
-        static MISSES: RefCell<(usize, Misses)> = RefCell::new((0, FxHashMap::default()));
-    }
-
-    pub(super) fn contains(ty: &Rc<TypeObject>, name: &str) -> bool {
-        MISSES.with(|m| {
-            m.borrow()
-                .1
-                .get(&(Rc::as_ptr(ty) as usize))
-                .is_some_and(|(w, names)| w.strong_count() > 0 && names.contains(name))
-        })
-    }
-
-    pub(super) fn insert(ty: &Rc<TypeObject>, name: &str) {
-        MISSES.with(|m| {
-            let mut m = m.borrow_mut();
-            let (count, map) = &mut *m;
-            if *count >= CAP {
-                *count = 0;
-                map.clear();
-            }
-            let entry = map
-                .entry(Rc::as_ptr(ty) as usize)
-                .or_insert_with(|| (Rc::downgrade(ty), FxHashSet::default()));
-            if entry.1.insert(name.into()) {
-                *count += 1;
-            }
-        });
-    }
-}
-
 /// Whether `obj.name` certainly raises `AttributeError` without running
 /// any code: an ordinary instance under the default attribute protocol,
 /// with no class attribute of that name and none in its `__dict__`. A
@@ -69556,8 +69506,13 @@ pub(crate) fn attr_name_of(o: &Object) -> Option<String> {
     }
 }
 
-fn attr_name_arg(o: Option<&Object>) -> Option<String> {
-    o.and_then(attr_name_of)
+/// A `getattr`/`hasattr` name argument (see [`attr_name_of`]), borrowing a
+/// plain `str`.
+fn attr_name_cow(o: Option<&Object>) -> Option<std::borrow::Cow<'_, str>> {
+    match o? {
+        Object::Str(s) => Some(std::borrow::Cow::Borrowed(&**s)),
+        other => attr_name_of(other).map(std::borrow::Cow::Owned),
+    }
 }
 
 /// `True` when `o` is a `StopAsyncIteration` instance (or one of
@@ -75327,17 +75282,23 @@ const INST_DICT: u64 = 0x5ca1_a770_0000_0009;
 /// protocol, the instance dict, after which the site remembers the class.
 /// `None` for any other name or shape. Runs no code (the dict may be
 /// created).
-#[inline(never)]
+#[inline(always)]
 fn inst_dict_attr(
     code: &CodeObject,
     inst: &PyInstance,
     pc: usize,
     name_idx: u32,
 ) -> Option<Object> {
-    if code.names.get(name_idx as usize)?.as_str() != "__dict__"
-        || inst.native.get().is_some()
-        || inst.c_body.get() != 0
-    {
+    if code.names.get(name_idx as usize)?.as_str() != "__dict__" {
+        return None;
+    }
+    inst_dict_attr_named(code, inst, pc)
+}
+
+/// [`inst_dict_attr`] once the name is `__dict__`.
+#[inline(never)]
+fn inst_dict_attr_named(code: &CodeObject, inst: &PyInstance, pc: usize) -> Option<Object> {
+    if inst.native.get().is_some() || inst.c_body.get() != 0 {
         return None;
     }
     let cls = inst.cls_raw();
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 94a080c4..1875b626 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -166,6 +166,11 @@ pub(crate) enum Verdict {
     /// attribute access and setting, and no class attribute in the way
     /// but a function or a plain value (see `Interpreter::leaf_store_attr`).
     PlainStore = 5,
+    /// A builtin class answered `name` with `AttributeError` (recorded
+    /// after the lookup, for `hasattr`; asyncio's `isfuture` asks
+    /// `hasattr(type(coro), '_asyncio_future_blocking')` for every
+    /// coroutine it is handed). See [`TypeObject::verdict_record`].
+    TypeAttrMissing = 6,
 }
 
 pub struct AttrVersion(std::sync::atomic::AtomicU64);
@@ -2221,6 +2226,37 @@ impl TypeObject {
         }
     }
 
+    /// The verdict [`Self::verdict_record`] recorded for `name` under the
+    /// class's current version, if any.
+    #[inline]
+    pub(crate) fn verdict_cached(&self, family: Verdict, name: &str) -> Option<bool> {
+        let ty = std::ptr::from_ref::<TypeObject>(self) as usize;
+        let ver = self.attr_version.get() | (family as u64) << 56;
+        let h = type_cache::name_hash(name) ^ (family as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
+        match type_cache::probe(ty, ver, name, h)? {
+            type_cache::Hit::Absent => Some(true),
+            type_cache::Hit::At { .. } => Some(false),
+        }
+    }
+
+    /// Record a verdict observed (rather than computed, as
+    /// [`Self::memo_verdict`] does) for `name` under the class's current
+    /// version; the cache may forget it.
+    pub(crate) fn verdict_record(&self, family: Verdict, name: &str, verdict: bool) {
+        let ty = std::ptr::from_ref::<TypeObject>(self) as usize;
+        let ver = self.attr_version.get() | (family as u64) << 56;
+        let h = type_cache::name_hash(name) ^ (family as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
+        let hit = if verdict {
+            type_cache::Hit::Absent
+        } else {
+            type_cache::Hit::At {
+                mro_idx: 0,
+                dict_idx: 0,
+            }
+        };
+        type_cache::fill(ty, ver, name, h, hit);
+    }
+
     /// The plain-`str`-keys MRO walk behind [`Self::lookup`] and
     /// [`Self::lookup_with_owner`], fronted by the per-thread
     /// [`type_cache`]. Returns the value and the MRO index of its owner.
diff --git a/tests/regrtest/test_attribute_slow_arms.py b/tests/regrtest/test_attribute_slow_arms.py
index d3233259..3d5dd3ff 100644
--- a/tests/regrtest/test_attribute_slow_arms.py
+++ b/tests/regrtest/test_attribute_slow_arms.py
@@ -503,6 +503,31 @@ def __class__(self):
         self.assertIs(weakref.proxy(c).__class__, C)
 
 
+class BuiltinTypeHasattrTest(unittest.TestCase):
+    def test_repeated_misses(self):
+        for _ in range(N):
+            self.assertFalse(hasattr(int, "_asyncio_future_blocking"))
+            self.assertTrue(hasattr(int, "from_bytes"))
+            self.assertEqual(getattr(int, "nope", 7), 7)
+
+    def test_struct_sequence_attribute_set(self):
+        import os
+
+        cls = type(os.stat("."))
+        for _ in range(N):
+            self.assertFalse(hasattr(cls, "weavepy_probe"))
+        try:
+            cls.weavepy_probe = 1
+        except TypeError:
+            return
+        self.assertTrue(hasattr(cls, "weavepy_probe"))
+        try:
+            del cls.weavepy_probe
+        except TypeError:
+            return
+        self.assertFalse(hasattr(cls, "weavepy_probe"))
+
+
 class Base:
     def __new__(cls, *args):
         return super().__new__(cls)

From 9cf936dafba8816adbfba69e2c450e84bce693d8 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 01:27:40 -0700
Subject: [PATCH 143/173] perf: raise at a failing attribute site without
 trying every lane

A LOAD_ATTR site cooling down after a failed load (`try: o.missing
except AttributeError:`) ran each cached-read lane in the compiled
code's helper, then the property and data-descriptor lanes, before the
full handler's certainly-missing check raised. At such a site an
instance receiver now goes straight to that check, which runs ahead of
the getter lanes; a receiver that does have the attribute still loads
through the full handler, which also counts the site's cooldown down.
The AttributeError's message borrows the class's name instead of
copying it.

Measured (instructions per iteration, host, PYTHONHASHSEED=0):
  try: o.missing except AttributeError    5953 -> 4573 (CPython 4276)
  try: o._s (unset slot) except ...        9970 -> 8937 (CPython 4983)
Callgrind totals (per call) against the previous commit: exceptions
1199.5M -> 1173.7M (-2.1%), stdlib_utils unchanged.
---
 crates/weavepy-vm/src/error.rs     | 16 ++++++++++++++--
 crates/weavepy-vm/src/frame_jit.rs | 18 ++++++++++++++++++
 crates/weavepy-vm/src/lib.rs       | 15 ++++++++-------
 3 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/crates/weavepy-vm/src/error.rs b/crates/weavepy-vm/src/error.rs
index 2616b541..ebb4ed63 100644
--- a/crates/weavepy-vm/src/error.rs
+++ b/crates/weavepy-vm/src/error.rs
@@ -297,10 +297,22 @@ pub fn attribute_error_named(obj: &Object, name: &str) -> RuntimeError {
     // Built in one step (message, `args` and both fields): a failing
     // lookup under `getattr(o, n, default)` or `except AttributeError`
     // is a hot path.
-    let type_name = obj.type_name_owned();
+    // (An instance's class name is borrowed, not copied.)
+    let cls = match obj {
+        Object::Instance(inst) => Some(inst.cls()),
+        _ => None,
+    };
+    let owned;
+    let type_name: &str = match &cls {
+        Some(c) if c.c_tp_name.get().is_none() => &c.name,
+        _ => {
+            owned = obj.type_name_owned();
+            &owned
+        }
+    };
     let mut message = String::with_capacity(type_name.len() + name.len() + 30);
     message.push('\'');
-    message.push_str(&type_name);
+    message.push_str(type_name);
     message.push_str("' object has no attribute '");
     message.push_str(name);
     message.push('\'');
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index d42155a5..4af9bf72 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3800,6 +3800,18 @@ fn slot_note(code: &CodeObject, recv: &Object, pc: usize, cache: *const SlotCach
     }
 }
 
+/// Whether the `LOAD_ATTR` at `pc` is cooling down after a failed load
+/// (an odd count; see `Interpreter::specialized_load_attr`): its cached
+/// reads are skipped for an instance, whose miss the full handler raises
+/// without its resolution passes (`try: o.x except AttributeError:`).
+#[inline(always)]
+fn failing_site(code: &CodeObject, pc: usize) -> bool {
+    matches!(
+        code.caches.get(pc as u32),
+        weavepy_compiler::InlineCache::Cooldown(n) if n & 1 == 1
+    )
+}
+
 /// `LOAD_ATTR` (the `pc`th instruction of `code`) on the receiver at
 /// stack slot `at`, as the core loop's arm runs it (and, for an instance
 /// or module, the leaf arm's cached reads): `0` the value in place of the
@@ -3826,6 +3838,9 @@ unsafe extern "C" fn h_stack_attr(
             crate::drop_hot(std::mem::replace(&mut *at, v));
             return 0;
         }
+        if matches!(recv, Object::Instance(_)) && failing_site(code, pc) {
+            return 1;
+        }
         let arg = code.instructions[pc].arg;
         let v = match recv {
             Object::Instance(inst) if inst.cls_raw().native_kind.get() != 0 => {
@@ -3933,6 +3948,9 @@ unsafe extern "C" fn h_local_attr(
             dst.write(v);
             return 0;
         }
+        if matches!(recv, Object::Instance(_)) && failing_site(code, pc) {
+            return 1;
+        }
         let arg = code.instructions[pc].arg;
         let v = match recv {
             Object::Instance(inst) if inst.cls_raw().native_kind.get() != 0 => {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 47520787..71b0c501 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -48169,10 +48169,18 @@ impl Interpreter {
         let frame = unsafe { &mut *sw.cur };
         let frame_field = frame.code.instructions[pc].op == OpCode::LoadAttr
             && matches!(frame.stack.last(), Some(Object::Frame(_) | Object::Code(_)));
+        // A site cooling down after a failed load (an odd count; see
+        // `specialized_load_attr`'s `Cooldown` arm) asks whether the
+        // attribute is certainly missing, ahead of the getter lanes.
+        let failing = matches!(
+            frame.code.caches.get(pc as u32),
+            weavepy_compiler::InlineCache::Cooldown(n) if n & 1 == 1
+        );
         // A property's or a Python data descriptor's getter runs as an
         // inline activation, as the core loop's own arm runs it.
         if frame.code.instructions[pc].op == OpCode::LoadAttr
             && !frame_field
+            && !failing
             && (self.core_property(sw) || self.core_getitem(sw, true))
         {
             return Ok(());
@@ -48198,13 +48206,6 @@ impl Interpreter {
         }
         frame.pc = pc as u32 + 1;
         let pending = self.core_pending_enter(sw, frame, pc);
-        // (Only a site cooling down after a failed load, an odd count, asks
-        // whether the attribute is certainly missing; see
-        // `specialized_load_attr`'s `Cooldown` arm.)
-        let failing = matches!(
-            frame.code.caches.get(pc as u32),
-            weavepy_compiler::InlineCache::Cooldown(n) if n & 1 == 1
-        );
         let r = match ins.op {
             OpCode::LoadAttr if failing => match self.load_attr_certain_miss(frame, ins.arg) {
                 Some(err) => Err(err),

From 0bbee25781b6d8f9e5672ccccd33eedb292bdc2c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:01:23 -0700
Subject: [PATCH 144/173] perf: skip the property lanes at failing attribute
 sites in the core loop

The core loop's helper exit and the quiet loop's slow step tried a
property getter (inline activation and lean evaluation) for every
LOAD_ATTR before stepping it, resolving the name on the receiver's class
each time. At a site cooling down after a failed load they are now
skipped, so the step's certainly-missing check raises without them (the
compiled code's helpers already skip them, see the previous commit).

Callgrind totals (per call) against the previous commit: exceptions
1173.4M -> 1143.7M (-2.5%), deepcopy unchanged.
---
 crates/weavepy-vm/src/frame_jit.rs | 16 ++--------------
 crates/weavepy-vm/src/lib.rs       | 26 ++++++++++++++++++++------
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 4af9bf72..88544eec 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3800,18 +3800,6 @@ fn slot_note(code: &CodeObject, recv: &Object, pc: usize, cache: *const SlotCach
     }
 }
 
-/// Whether the `LOAD_ATTR` at `pc` is cooling down after a failed load
-/// (an odd count; see `Interpreter::specialized_load_attr`): its cached
-/// reads are skipped for an instance, whose miss the full handler raises
-/// without its resolution passes (`try: o.x except AttributeError:`).
-#[inline(always)]
-fn failing_site(code: &CodeObject, pc: usize) -> bool {
-    matches!(
-        code.caches.get(pc as u32),
-        weavepy_compiler::InlineCache::Cooldown(n) if n & 1 == 1
-    )
-}
-
 /// `LOAD_ATTR` (the `pc`th instruction of `code`) on the receiver at
 /// stack slot `at`, as the core loop's arm runs it (and, for an instance
 /// or module, the leaf arm's cached reads): `0` the value in place of the
@@ -3838,7 +3826,7 @@ unsafe extern "C" fn h_stack_attr(
             crate::drop_hot(std::mem::replace(&mut *at, v));
             return 0;
         }
-        if matches!(recv, Object::Instance(_)) && failing_site(code, pc) {
+        if matches!(recv, Object::Instance(_)) && crate::failing_attr_site(code, pc) {
             return 1;
         }
         let arg = code.instructions[pc].arg;
@@ -3948,7 +3936,7 @@ unsafe extern "C" fn h_local_attr(
             dst.write(v);
             return 0;
         }
-        if matches!(recv, Object::Instance(_)) && failing_site(code, pc) {
+        if matches!(recv, Object::Instance(_)) && crate::failing_attr_site(code, pc) {
             return 1;
         }
         let arg = code.instructions[pc].arg;
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 71b0c501..a8be025b 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -7933,16 +7933,20 @@ impl Interpreter {
                         return FrameEv::Call(act);
                     }
                 }
+                // (A site whose loads keep failing goes straight to the full
+                // handler, which raises a certain miss without the lanes.)
+                let attr_failing =
+                    op_before == Some(OpCode::LoadAttr) && failing_attr_site(&frame.code, cur_pc);
                 // `inst.prop` for a plain `property`: its getter runs as an
                 // inline activation, like an inline call.
-                if op_before == Some(OpCode::LoadAttr) && self.inline_calls_ok() {
+                if op_before == Some(OpCode::LoadAttr) && !attr_failing && self.inline_calls_ok() {
                     if let Some(act) = self.try_inline_property(frame, shell, cur_pc) {
                         return FrameEv::Call(act);
                     }
                 }
                 // Otherwise as a lean activation; `for x in gen` resumes
                 // the generator as one.
-                if matches!(op_before, Some(OpCode::LoadAttr | OpCode::ForIter)) {
+                if matches!(op_before, Some(OpCode::LoadAttr | OpCode::ForIter)) && !attr_failing {
                     let loaded = if op_before == Some(OpCode::LoadAttr) {
                         self.try_lean_property(frame, shell, snap_gen, cur_pc)
                     } else {
@@ -10950,6 +10954,7 @@ impl Interpreter {
                 .instructions
                 .get(pc)
                 .is_some_and(|i| i.op == OpCode::LoadAttr)
+                && !failing_attr_site(&frame.code, pc)
             {
                 if let Some((getter, code)) = Self::cached_property_getter(frame, pc) {
                     // SAFETY: as above.
@@ -48172,10 +48177,7 @@ impl Interpreter {
         // A site cooling down after a failed load (an odd count; see
         // `specialized_load_attr`'s `Cooldown` arm) asks whether the
         // attribute is certainly missing, ahead of the getter lanes.
-        let failing = matches!(
-            frame.code.caches.get(pc as u32),
-            weavepy_compiler::InlineCache::Cooldown(n) if n & 1 == 1
-        );
+        let failing = failing_attr_site(&frame.code, pc);
         // A property's or a Python data descriptor's getter runs as an
         // inline activation, as the core loop's own arm runs it.
         if frame.code.instructions[pc].op == OpCode::LoadAttr
@@ -75266,6 +75268,18 @@ const LAID_OUT_SLOT: u64 = 0x5ca1_a770_0000_0006;
 /// [`Interpreter::core_delete_attr`]).
 const DELETE_OWN_ATTR: u64 = 0x5ca1_a770_0000_0007;
 
+/// Whether the `LOAD_ATTR` at `pc` is cooling down after a failed load
+/// (an odd count; see `Interpreter::specialized_load_attr`'s `Cooldown`
+/// arm): its lanes are skipped for the full handler, whose certainly
+/// missing check raises (`try: o.x except AttributeError:`).
+#[inline(always)]
+pub(crate) fn failing_attr_site(code: &CodeObject, pc: usize) -> bool {
+    matches!(
+        code.caches.get(pc as u32),
+        weavepy_compiler::InlineCache::Cooldown(n) if n & 1 == 1
+    )
+}
+
 /// A `LOAD_ATTR` site's stamp tag for a module attribute: the first word
 /// is the module dict's mutation stamp, the bits the entry's position.
 /// A stamp names one key layout of one dict (values stored in place keep

From 7840802c34a90f951eaf6864022835ef4aca6a9a Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:14:42 -0700
Subject: [PATCH 145/173] perf: read a ChainMap's maps without resolving the
 name each lookup

The native `ChainMap.__getitem__` resolved `maps` on the receiver's
class through the type cache and then searched the instance's split
layout for it by name on every subscript. The class's answer is now
cached on the class under its version (`class_lacks`), and the
instance's `maps` is read with the name's hash computed once and its
position in the split layout remembered.

Measured (instructions per iteration, host, PYTHONHASHSEED=0):
  c['a']; c['b'] on a two-map ChainMap   2226 -> 1692 (CPython 3547)
---
 .../src/stdlib/collections_native.rs          | 54 +++++++++++++++++--
 tests/regrtest/test_attribute_slow_arms.py    | 33 ++++++++++++
 2 files changed, 83 insertions(+), 4 deletions(-)

diff --git a/crates/weavepy-vm/src/stdlib/collections_native.rs b/crates/weavepy-vm/src/stdlib/collections_native.rs
index 504e8cbb..f1bcf3e8 100644
--- a/crates/weavepy-vm/src/stdlib/collections_native.rs
+++ b/crates/weavepy-vm/src/stdlib/collections_native.rs
@@ -1172,6 +1172,46 @@ fn chainmap_getitem(args: &[Object]) -> Result<Object, RuntimeError> {
     interp.call_object(missing, std::slice::from_ref(key), &[])
 }
 
+/// The class cache key for "the class has no `maps` attribute" (see
+/// `Interpreter::class_lacks`).
+static MAPS_CLASS_KEY: u8 = 0;
+
+/// A ChainMap instance's own `maps` list, read without running code: the
+/// name's hash computed once, and its position in a split layout
+/// remembered (see [`crate::inst_dict::PosMemo`]).
+fn chainmap_maps_of(inst: &crate::types::PyInstance) -> Option<Rc<RefCell<Vec<Object>>>> {
+    struct MapsName {
+        hash: i64,
+        memo: crate::inst_dict::PosMemo,
+    }
+    static NAME: std::sync::OnceLock<MapsName> = std::sync::OnceLock::new();
+    let n = NAME.get_or_init(|| MapsName {
+        hash: crate::object::py_str_hash("maps"),
+        memo: crate::inst_dict::PosMemo::new(),
+    });
+    let v = match inst.dict.published() {
+        Some(d) => d
+            .try_borrow()
+            .ok()?
+            .get(&crate::object::StrKeyHashed {
+                s: "maps",
+                hash: n.hash,
+            })
+            .cloned(),
+        None => inst
+            .dict
+            .split_cell()
+            .try_borrow()
+            .ok()?
+            .get_memo("maps", n.hash, &n.memo)
+            .cloned(),
+    };
+    match v? {
+        Object::List(l) => Some(l),
+        _ => None,
+    }
+}
+
 /// [`chainmap_getitem`]'s leaf half (see `leaf_builtins::Fast`): a plain
 /// key found by exact native equality in a list of exact dicts held in
 /// the instance's own `maps` attribute. A key no map holds (the
@@ -1180,12 +1220,18 @@ fn chainmap_getitem_leaf(args: &[Object]) -> Option<Result<Object, RuntimeError>
     let [Object::Instance(inst), key] = args else {
         return None;
     };
-    if crate::object::exotic_str_keys_possible() || inst.cls_raw().lookup("maps").is_some() {
+    // (The class's answer is cached on it under its version, and the
+    // instance's slot for the name remembered; see `chainmap_maps_of`.)
+    if crate::object::exotic_str_keys_possible()
+        || !crate::Interpreter::class_lacks(
+            inst.cls_raw(),
+            std::ptr::addr_of!(MAPS_CLASS_KEY) as usize,
+            "maps",
+        )
+    {
         return None;
     }
-    let Object::List(maps) = inst.attr_get_str("maps")? else {
-        return None;
-    };
+    let maps = chainmap_maps_of(inst)?;
     let probe = crate::object::LeafProbe::new(key)?;
     let maps = maps.try_borrow().ok()?;
     for mapping in maps.iter() {
diff --git a/tests/regrtest/test_attribute_slow_arms.py b/tests/regrtest/test_attribute_slow_arms.py
index 3d5dd3ff..1124dd94 100644
--- a/tests/regrtest/test_attribute_slow_arms.py
+++ b/tests/regrtest/test_attribute_slow_arms.py
@@ -528,6 +528,39 @@ def test_struct_sequence_attribute_set(self):
         self.assertFalse(hasattr(cls, "weavepy_probe"))
 
 
+class ChainMapTest(unittest.TestCase):
+    def test_maps_changes(self):
+        from collections import ChainMap
+
+        c = ChainMap({"a": 1}, {"b": 2})
+        for _ in range(N):
+            self.assertEqual(c["a"] + c["b"], 3)
+        c.maps = [{"a": 10}]
+        self.assertEqual(c["a"], 10)
+        with self.assertRaises(KeyError):
+            c["b"]
+        c.maps.append({"b": 20})
+        self.assertEqual(c["b"], 20)
+
+    def test_subclass_with_class_level_maps(self):
+        from collections import ChainMap
+
+        class Odd(ChainMap):
+            pass
+
+        o = Odd({"a": 1})
+        for _ in range(N):
+            self.assertEqual(o["a"], 1)
+        o.__dict__["maps"] = [{"a": 5}]
+        self.assertEqual(o["a"], 5)
+        Odd.maps = property(lambda self: [{"a": 7}], lambda self, v: None)
+        try:
+            self.assertEqual(o["a"], 7)
+        finally:
+            del Odd.maps
+        self.assertEqual(o["a"], 5)
+
+
 class Base:
     def __new__(cls, *args):
         return super().__new__(cls)

From 8a298eb989de5b13378dc41908f877d76e4dd039 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:36:51 -0700
Subject: [PATCH 146/173] perf: remember a stored callable's slot at its
 method-form load site

`self.fn(...)` where the instance holds `fn` itself (mini_interpreter's
`BinOp.eval`) resolved the name on the class and searched the split
layout on every load. The site now remembers the value's split position
for the receiver's class (the field slot an instance-field LOAD_ATTR
keeps), under the class's version, so a later load is a version compare
and an indexed read.

Callgrind totals (per call) against the ChainMap commit's parent:
mini_interpreter 1873.1M -> 1837.3M (-1.9%); collections_mix 556.1M ->
547.7M (-1.5%, with the ChainMap change).
---
 crates/weavepy-vm/src/frame_jit.rs         |  2 +-
 crates/weavepy-vm/src/lib.rs               | 27 ++++++++++++++++++----
 tests/regrtest/test_attribute_slow_arms.py | 27 ++++++++++++++++++++++
 3 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 88544eec..1459dcc2 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3533,7 +3533,7 @@ unsafe extern "C" fn h_load_method(
                 // An attribute the instance holds itself (a stored callable):
                 // the value with an empty self slot, as the core loop's arm.
                 if crate::inst_may_shadow(inst, code, name) {
-                    return u32::from(!Interpreter::core_value_method(code, name, top));
+                    return u32::from(!Interpreter::core_value_method(code, name, pc, top));
                 }
                 let f = match ms.get_held(ver) {
                     Some(f) => {
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index a8be025b..f5012e74 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -14585,7 +14585,7 @@ impl Interpreter {
                             if inst_may_shadow(inst, code, ins.arg) {
                                 // SAFETY: `len < cap`: `top` and the slot
                                 // above it.
-                                if unsafe { Self::core_value_method(code, ins.arg, top) } {
+                                if unsafe { Self::core_value_method(code, ins.arg, pc, top) } {
                                     len += 1;
                                     last = pc;
                                     pc += 1;
@@ -18941,20 +18941,37 @@ impl Interpreter {
     ///
     /// `top` is the core loop's top stack slot (an instance), and the slot
     /// above it is free.
+    ///
+    /// The site at `pc` remembers the value's split position for the
+    /// receiver's class (see [`FieldSlot`]), as an instance-field
+    /// `LOAD_ATTR` does, so later loads skip the resolution.
     #[inline(never)]
     pub(crate) unsafe fn core_value_method(
         code: &CodeObject,
         name_idx: u32,
+        pc: usize,
         top: *mut Object,
     ) -> bool {
         // SAFETY: the caller's contract.
         let Object::Instance(inst) = (unsafe { &*top }) else {
             return false;
         };
-        let Some((LeafAttr::Value(v), _)) =
-            Self::leaf_resolve_instance_attr_ix(code, inst, name_idx)
-        else {
-            return false;
+        let ext = code_vm_ext(code);
+        // SAFETY: a read between two instructions (see `GilCell::peek`).
+        let held = ext.and_then(|e| unsafe { field_slot_hit(e, pc, inst) });
+        let v = match held {
+            Some(v) => Self::clone_operand(v),
+            None => {
+                let Some((LeafAttr::Value(v), ix)) =
+                    Self::leaf_resolve_instance_attr_ix(code, inst, name_idx)
+                else {
+                    return false;
+                };
+                if let (Some(ext), Some(ix), None) = (ext, ix, inst.dict.published()) {
+                    field_slot_note(ext, code, pc, inst, ix);
+                }
+                v
+            }
         };
         // SAFETY: as above.
         if !Self::core_droppable(unsafe { &*top }) {
diff --git a/tests/regrtest/test_attribute_slow_arms.py b/tests/regrtest/test_attribute_slow_arms.py
index 1124dd94..a9c4bb76 100644
--- a/tests/regrtest/test_attribute_slow_arms.py
+++ b/tests/regrtest/test_attribute_slow_arms.py
@@ -561,6 +561,33 @@ class Odd(ChainMap):
         self.assertEqual(o["a"], 5)
 
 
+class StoredCallableTest(unittest.TestCase):
+    def test_stored_callable_method_form(self):
+        class Node:
+            def fn(self, x):
+                return "method"
+
+            def __init__(self, fn):
+                self.fn = fn
+
+            def run(self, x):
+                return self.fn(x)
+
+        a = Node(lambda x: x + 1)
+        b = Node(lambda x: x * 2)
+        for i in range(N):
+            self.assertEqual(a.run(i), i + 1)
+            self.assertEqual(b.run(i), 2 * i)
+        a.fn = lambda x: -x
+        self.assertEqual(a.run(3), -3)
+        del a.fn
+        self.assertEqual(a.run(3), "method")
+        b.__dict__  # publishes the dict
+        self.assertEqual(b.run(3), 6)
+        Node.fn = property(lambda self: (lambda x: "prop"))
+        self.assertEqual(b.run(3), "prop")
+
+
 class Base:
     def __new__(cls, *args):
         return super().__new__(cls)

From 563243486d34aaa446ce6e74b94ba8d4ec0d3923 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 18 Oct 2026 22:09:09 -0700
Subject: [PATCH 147/173] fix: keep keyword permutations off slot 15

A keyword call's permutation packs each keyword's parameter slot into a
4-bit nibble, and nibble 15 is KW_TO_VARKW ("into the **kwargs dict"),
which the binders read without checking the callee. kw_perm allowed 16
parameter slots for callees without **kwargs, so a keyword aimed at
slot 15 was taken for a **kwargs keyword and dropped. Since the call
lane began binding keywords through kw_perm, this left the parameter
unbound: dataclasses.make_dataclass (2 positional + 14 keyword-only
parameters) raised UnboundLocalError when given decorator=.

The permutation now covers at most 15 parameter slots for every callee;
wider signatures take the generic binder, as they did for **kwargs
callees already. The CALL_KW site re-check gets the same limit.
---
 crates/weavepy-vm/src/lib.rs                 |  4 +-
 crates/weavepy-vm/src/specialize.rs          | 17 +++---
 tests/regrtest/test_kw_binding_slot_limit.py | 60 ++++++++++++++++++++
 3 files changed, 71 insertions(+), 10 deletions(-)
 create mode 100644 tests/regrtest/test_kw_binding_slot_limit.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index f5012e74..f8e0952f 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -9441,8 +9441,8 @@ impl Interpreter {
         let npos = code.arg_count as usize;
         let total = npos + code.kwonly_count as usize;
         if specialize::rc_id(f) != func_id
-            || total > 16
-            || (code.has_varkeywords && (total > 15 || code.varnames.len() <= total))
+            || total > 15
+            || (code.has_varkeywords && code.varnames.len() <= total)
             || eff_argc > npos
             || code.is_generator
             || code.is_coroutine
diff --git a/crates/weavepy-vm/src/specialize.rs b/crates/weavepy-vm/src/specialize.rs
index 825b74b3..5fbebacf 100644
--- a/crates/weavepy-vm/src/specialize.rs
+++ b/crates/weavepy-vm/src/specialize.rs
@@ -503,7 +503,8 @@ pub fn attempt_specialize_call(callable: &Object, argc: usize) -> InlineCache {
 }
 
 /// The `CallPyKwNames` permutation nibble of a keyword a `**kwargs`
-/// callee collects into its dict (parameter slots stay below 15 there).
+/// callee collects into its dict. Its consumers read it without checking
+/// the callee, so no parameter slot may be 15 (see [`kw_perm`]).
 pub const KW_TO_VARKW: u32 = 0xF;
 
 /// Decide on a `CALL_KW` specialization (RFC 0069 WS3): a plain Python
@@ -513,7 +514,7 @@ pub const KW_TO_VARKW: u32 = 0xF;
 /// positional count, and every slot left uncovered is filled by a
 /// trailing default. The kw→slot permutation packs 4 bits per keyword
 /// into `perm`; anything that doesn't fit (more than 8 keywords, more
-/// than 16 parameter slots) stays generic. Binder *errors* (multiple
+/// than 15 parameter slots) stays generic. Binder *errors* (multiple
 /// values, unexpected keyword, posonly-as-keyword, missing arguments)
 /// all fail these checks, so the generic binder keeps owning the
 /// error messages.
@@ -546,7 +547,7 @@ pub fn attempt_specialize_call_kw(
 /// `names`, and the parameter slots the call fills: each keyword's slot in
 /// 4 bits (`KW_TO_VARKW` for one a `**kwargs` callee collects). Each name
 /// is tried first at the parameter its position gives it. `None` for any
-/// call the generic binder must take: `*args`, more than 8 keywords or 16
+/// call the generic binder must take: `*args`, more than 8 keywords or 15
 /// parameter slots, replaced defaults, a keyword without a distinct
 /// keyword-bindable parameter (or `**kwargs`), a parameter left without a
 /// default.
@@ -563,11 +564,11 @@ pub fn kw_perm<'a>(
     if code.has_varargs
         || (f.defaults_maybe_overridden()
             && (f.slot("__defaults__").is_some() || f.slot("__kwdefaults__").is_some()))
-        || total > 16
-        // A `**kwargs` callee reserves nibble 15 for the keywords it
-        // collects (see `KW_TO_VARKW`), and its dict slot follows the
-        // parameters.
-        || (code.has_varkeywords && (total > 15 || code.varnames.len() <= total))
+        // Nibble 15 is `KW_TO_VARKW`, which the binders take as "into
+        // the `**kwargs` dict" for any callee: slot 15 can't be encoded.
+        || total > 15
+        // A `**kwargs` callee's dict slot follows the parameters.
+        || (code.has_varkeywords && code.varnames.len() <= total)
         || kwc == 0
         || kwc > 8
         || argc > npos
diff --git a/tests/regrtest/test_kw_binding_slot_limit.py b/tests/regrtest/test_kw_binding_slot_limit.py
new file mode 100644
index 00000000..745ac939
--- /dev/null
+++ b/tests/regrtest/test_kw_binding_slot_limit.py
@@ -0,0 +1,60 @@
+"""Keyword arguments bind to every parameter slot of wide signatures.
+
+The keyword-call permutation packs each keyword's parameter slot into a
+4-bit nibble whose value 15 means "into `**kwargs`"; a callee with 16
+parameter slots once lost a keyword aimed at slot 15
+(`dataclasses.make_dataclass(..., decorator=...)` raised
+UnboundLocalError). Exercised at module level, in a loop, and in a hot
+function, at signature widths around the limit.
+"""
+
+import dataclasses
+
+
+def make(npos, nkw, varkw=False):
+    pos = [f"a{i}" for i in range(npos)]
+    kws = [f"k{i}=None" for i in range(nkw)]
+    params = pos + (["*"] + kws if kws else [])
+    if varkw:
+        params.append("**rest")
+    names = [f"a{i}" for i in range(npos)] + [f"k{i}" for i in range(nkw)]
+    items = [f"{n!r}: {n}" for n in names] + (["'rest': rest"] if varkw else [])
+    ns = {}
+    exec(f"def f({', '.join(params)}):\n    return {{{', '.join(items)}}}\n", ns)
+    return ns["f"]
+
+
+def call_with(f, npos, name, value):
+    """`f(0, 1, ..., name=value)` from a literal keyword call site."""
+    ns = {"f": f}
+    pos = "".join(f"{i}, " for i in range(npos))
+    exec(f"def run():\n    return f({pos}{name}={value!r})\n", ns)
+    return ns["run"]
+
+
+for npos, nkw in [(0, 15), (0, 16), (0, 17), (1, 15), (2, 13), (2, 14), (2, 15), (3, 14)]:
+    for varkw in (False, True):
+        f = make(npos, nkw, varkw)
+        for j in (0, nkw // 2, nkw - 1):
+            name = f"k{j}"
+            run = call_with(f, npos, name, "X")
+            for _ in range(300):
+                got = run()
+                assert got[name] == "X", (npos, nkw, varkw, name, got)
+                for i in range(nkw):
+                    if i != j:
+                        assert got[f"k{i}"] is None, (npos, nkw, varkw, name, got)
+        if varkw:
+            got = call_with(f, npos, "extra", 1)()
+            assert got["rest"] == {"extra": 1}, got
+
+
+def deco(cls, **kw):
+    return dataclasses.dataclass(cls, **kw)
+
+
+for _ in range(300):
+    C = dataclasses.make_dataclass("C", [("x", int)], decorator=deco)
+    assert repr(C(1)) == "C(x=1)"
+
+print("kw binding slot limit: ok")

From 0610b53f575f907c137405953a9ee33e9023113c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 21:31:27 -0700
Subject: [PATCH 148/173] perf: release JIT compile contexts and pack frame
 exit constants

The frame, leaf-plan and tier-2 compilers cleared their Cranelift
function and builder contexts between compiles, which keeps every
buffer at the largest body's capacity for the rest of the process and
under each later compile's peak. They now drop both for empty ones
after each compile (and the frame compiler after a body it re-lowers
lean).

A frame JIT exit jumped to its shared tail with the stack depth, pc and
last instruction as three constants: about 600 of the 4,714 IR
instructions in chaos's Spline.__call__. One packed constant replaces
them, decoded in the tail. Spline.__call__ goes from 4,714 to 4,364 IR
instructions and Chaosgame.transform_point from 4,288 to 3,978; the
lean threshold still measures every exit as if written in line.

Peak RSS (quick build, warm caches): chaos 29.6 to 29.0 MB, go 27.9 to
27.5 MB, raytrace 26.3 to 25.7 MB. The suite's instruction geomean
moved +0.3%, within the quick build's layout noise.
---
 crates/weavepy-jit/src/engine.rs       | 17 +++++--
 crates/weavepy-vm/src/frame_jit.rs     | 69 ++++++++++++++++----------
 crates/weavepy-vm/src/leaf_plan/jit.rs | 12 +++--
 3 files changed, 63 insertions(+), 35 deletions(-)

diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs
index 48e4dd25..94c15cdc 100644
--- a/crates/weavepy-jit/src/engine.rs
+++ b/crates/weavepy-jit/src/engine.rs
@@ -583,6 +583,14 @@ impl JitEngine {
         })
     }
 
+    /// Drop the function and builder contexts for empty ones: clearing
+    /// them keeps their buffers' capacity, the largest function's size,
+    /// for the rest of the thread (and under the next compile's peak).
+    fn release(&mut self) {
+        self.ctx = self.module.make_context();
+        self.fbctx = FunctionBuilderContext::new();
+    }
+
     /// Analyze and compile a code object. `resolve` reports what each
     /// `LOAD_GLOBAL` name currently resolves to (see
     /// [`ResolvedGlobal`]); the caller must re-validate every resolution
@@ -1007,7 +1015,7 @@ impl JitEngine {
             inline_method,
         );
         if !sound {
-            self.module.clear_context(&mut self.ctx);
+            self.release();
             return Err(JitVerdict::NotConverged);
         }
         if std::env::var_os("WEAVEPY_JIT_CLIF_DUMP").is_some() {
@@ -1032,11 +1040,10 @@ impl JitEngine {
             let _ = cranelift_codegen::timing::take_current();
         }
         let t0 = stats.then(std::time::Instant::now);
-        self.module
-            .define_function(id, &mut self.ctx)
-            .map_err(|_| JitVerdict::NotConverged)?;
+        let defined = self.module.define_function(id, &mut self.ctx);
         let t1 = stats.then(std::time::Instant::now);
-        self.module.clear_context(&mut self.ctx);
+        self.release();
+        defined.map_err(|_| JitVerdict::NotConverged)?;
         self.module
             .finalize_definitions()
             .map_err(|_| JitVerdict::NotConverged)?;
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 1459dcc2..647e49d5 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -112,6 +112,12 @@ const MAX_FAILS: u8 = 32;
 /// The longest code compiled.
 const MAX_INSTRUCTIONS: usize = 4096;
 
+/// An exit's packed constant (see `Lower::exit`): the stack depth in its
+/// low bits, the pc from bit `EXIT_PC_SHIFT`, and the last instruction
+/// from bit `EXIT_LAST_SHIFT` (both below `MAX_INSTRUCTIONS`).
+const EXIT_PC_SHIFT: i64 = 20;
+const EXIT_LAST_SHIFT: i64 = 42;
+
 /// The running activation's state, as the core loop holds it: the native
 /// code reads it on entry and writes it back on exit.
 #[repr(C)]
@@ -6926,6 +6932,14 @@ unsafe impl Send for Engine {}
 static ENGINE: std::sync::Mutex<Option<Option<Engine>>> = std::sync::Mutex::new(None);
 
 impl Engine {
+    /// Drop the function and builder contexts for empty ones: clearing
+    /// them keeps their buffers' capacity, the largest body's size, for
+    /// the rest of the process (and under the next compile's peak).
+    fn release(&mut self) {
+        self.ctx = self.module.make_context();
+        self.fbctx = FunctionBuilderContext::new();
+    }
+
     fn new() -> Option<Engine> {
         let tags = tags()?;
         let mut flags = settings::builder();
@@ -7055,8 +7069,7 @@ fn compile_with(
         }
         if done && !lean && engine.ctx.func.dfg.num_insts() + size_in_line > lean_above() {
             lean = true;
-            engine.fbctx = FunctionBuilderContext::new();
-            engine.module.clear_context(&mut engine.ctx);
+            engine.release();
             engine.ctx.func.signature = signature.clone();
             continue;
         }
@@ -7064,8 +7077,7 @@ fn compile_with(
     };
     if !built {
         // An unfinished function leaves the builder's state behind.
-        engine.fbctx = FunctionBuilderContext::new();
-        engine.module.clear_context(&mut engine.ctx);
+        engine.release();
         return None;
     }
     if std::env::var("WEAVEPY_FRAME_JIT_DUMP").is_ok_and(|n| code.qualname.contains(&n)) {
@@ -7094,7 +7106,7 @@ fn compile_with(
             code.qualname
         );
     }
-    engine.module.clear_context(&mut engine.ctx);
+    engine.release();
     defined.ok()?;
     engine.module.finalize_definitions().ok()?;
     let func = engine.module.get_finalized_function(id);
@@ -7703,38 +7715,41 @@ impl<'a> Lower<'a> {
             Some(&(_, _, tail)) => tail,
             None => {
                 let tail = self.b.create_block();
-                let params = if last.is_some() { 3 } else { 2 };
-                for _ in 0..params {
-                    self.b.append_block_param(tail, types::I64);
-                }
+                self.b.append_block_param(tail, types::I64);
                 self.exit_tails.push((status, last.is_some(), tail));
                 tail
             }
         };
-        // In line, the status constant, the stores and the return less
-        // the jump.
-        self.tail_savings += if last.is_some() { 4 } else { 3 };
-        let l = self.b.ins().iconst(types::I64, self.depth as i64);
-        let pcv = self.b.ins().iconst(types::I64, pc as i64);
-        match last {
-            Some(last) => {
-                let lv = self.b.ins().iconst(types::I64, last as i64);
-                self.b.ins().jump(tail, &[l.into(), pcv.into(), lv.into()]);
-            }
-            None => {
-                self.b.ins().jump(tail, &[l.into(), pcv.into()]);
-            }
+        // In line, the depth, pc, last instruction and status constants,
+        // the stores and the return, less the packed constant and the
+        // jump.
+        self.tail_savings += if last.is_some() { 6 } else { 4 };
+        // The depth, pc and last instruction packed in one constant (see
+        // `EXIT_PC_SHIFT`): one value to the tail where three would each
+        // be an instruction and a block argument to allocate.
+        debug_assert!(self.depth < 1 << EXIT_PC_SHIFT);
+        debug_assert!(pc < 1 << (EXIT_LAST_SHIFT - EXIT_PC_SHIFT));
+        let mut packed = self.depth as i64 | (pc as i64) << EXIT_PC_SHIFT;
+        if let Some(last) = last {
+            packed |= (last as i64) << EXIT_LAST_SHIFT;
         }
+        let v = self.b.ins().iconst(types::I64, packed);
+        self.b.ins().jump(tail, &[v.into()]);
     }
 
     /// Fill the exit tails (see [`Self::exit`]).
     fn exit_tails(&mut self) {
-        for (status, _, tail) in std::mem::take(&mut self.exit_tails) {
+        for (status, last, tail) in std::mem::take(&mut self.exit_tails) {
             self.b.switch_to_block(tail);
-            let params = self.b.block_params(tail).to_vec();
-            self.b.ins().store(FLAGS, params[0], self.st, S_LEN);
-            self.b.ins().store(FLAGS, params[1], self.st, S_PC);
-            if let Some(&last) = params.get(2) {
+            let packed = self.b.block_params(tail)[0];
+            let field = (1 << (EXIT_LAST_SHIFT - EXIT_PC_SHIFT)) - 1;
+            let len = self.b.ins().band_imm(packed, (1 << EXIT_PC_SHIFT) - 1);
+            let pc = self.b.ins().ushr_imm(packed, EXIT_PC_SHIFT);
+            let pc = self.b.ins().band_imm(pc, field);
+            self.b.ins().store(FLAGS, len, self.st, S_LEN);
+            self.b.ins().store(FLAGS, pc, self.st, S_PC);
+            if last {
+                let last = self.b.ins().ushr_imm(packed, EXIT_LAST_SHIFT);
                 self.b.ins().store(FLAGS, last, self.st, S_LAST);
             }
             let s = self.b.ins().iconst(types::I32, i64::from(status));
diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs
index 31342a9a..53f3b533 100644
--- a/crates/weavepy-vm/src/leaf_plan/jit.rs
+++ b/crates/weavepy-vm/src/leaf_plan/jit.rs
@@ -898,6 +898,13 @@ unsafe impl Send for Engine {}
 static ENGINE: std::sync::Mutex<Option<Option<Engine>>> = std::sync::Mutex::new(None);
 
 impl Engine {
+    /// Drop the function and builder contexts for empty ones (see the
+    /// frame compiler's `Engine::release`).
+    fn release(&mut self) {
+        self.ctx = self.module.make_context();
+        self.fbctx = FunctionBuilderContext::new();
+    }
+
     fn new() -> Option<Engine> {
         if !tags_match() {
             return None;
@@ -1029,8 +1036,7 @@ fn compile_with(
     };
     let Some((depth, frames, keep, fields, globals, methods)) = lowered else {
         // An unfinished function leaves the builder's state behind.
-        engine.fbctx = FunctionBuilderContext::new();
-        engine.module.clear_context(&mut engine.ctx);
+        engine.release();
         return None;
     };
     let defined = engine.module.define_function(id, &mut engine.ctx);
@@ -1040,7 +1046,7 @@ fn compile_with(
             code.qualname
         );
     }
-    engine.module.clear_context(&mut engine.ctx);
+    engine.release();
     defined.ok()?;
     engine.module.finalize_definitions().ok()?;
     let func = engine.module.get_finalized_function(id);

From 69b332b621e096a297cb1b6488fd6744159749e9 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 23:49:21 -0700
Subject: [PATCH 149/173] perf: trim lean frame bodies' IR for compile memory

Cranelift's working memory grows with the IR: about 1 KB of resident
memory per IR instruction with the backtracking allocator, so a large
lean body's compile sets many small workloads' peak (TextWrapper.
_wrap_chunks, 8,285 IR instructions, raised text_processing's RSS by
9 MB while it compiled). Three changes cut what lean bodies emit:

- Every helper status that leaves the native code jumps to one shared
  tail (the raise test and the return) instead of repeating it.
- A lean body's CALL makes the choice between the direct call and the
  general one in a new helper, h_call_site, instead of a six-block
  decision in line (non-lean bodies keep it in line).
- A lean body tries an operator method directly (h_binop_dunder) only
  at a site that has run on instances; elsewhere a declined operation
  goes back to the core loop, which runs it.

Over the suite's compiles, lean bodies' IR falls 10.7%, their blocks
16.4% and their code 14.4% (non-lean: 1%, 2% and 4%); _wrap_chunks goes
to 7,260 IR instructions. The frame JIT's exit counts are unchanged on
every workload.

Peak RSS (quick build, warm caches): suite geomean 0.987x, with
text_processing 27.6 to 26.0 MB, stdlib_utils 32.6 to 31.4 MB,
tomllib_loads 27.9 to 27.1 MB and go 24.9 to 24.4 MB. Instructions per
call: geomean 0.999x.
---
 crates/weavepy-vm/src/frame_jit.rs | 99 ++++++++++++++++++++++++++++--
 1 file changed, 93 insertions(+), 6 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 647e49d5..34624412 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -4712,6 +4712,37 @@ unsafe extern "C" fn h_call_direct(
     }
 }
 
+/// `CALL` from a lean body (see [`Lower::call_op`]): a Python function's
+/// call tries [`h_call_direct`] first, unless its site found the callee
+/// without a native body and has retries left to count down; anything
+/// else goes through [`h_call`]. A lean body makes this choice here
+/// rather than in line: its code is large enough that compile memory
+/// matters more than the branches.
+unsafe extern "C" fn h_call_site(
+    st: *mut State,
+    code: *const CodeObject,
+    ext: *const CodeConstObjects,
+    pc_argc: u64,
+    len: u64,
+    site: *mut DirectSite,
+) -> u32 {
+    let argc = (pc_argc >> 32) as usize;
+    // SAFETY: as `h_call_direct`'s: the callee's slot is on the stack
+    // below the arguments and its self slot, and `site` is this `CALL`'s
+    // own cache.
+    unsafe {
+        let callee = &*(*st).stack.add(len as usize - argc - 2);
+        if matches!(callee, Object::Function(_)) {
+            let site = &mut *site;
+            if site.hot != 0 || site.retry == 0 {
+                return h_call_direct(st, code, ext, pc_argc, len, site);
+            }
+            site.retry -= 1;
+        }
+        h_call(st, code, ext, pc_argc & u64::from(u32::MAX), len)
+    }
+}
+
 /// Whether a site may evaluate a call of `f` (running `code`) with `nargs`
 /// positional arguments as a frameless leaf ([`leaf_site_call`]): a leaf
 /// with only positional parameters, at most eight, `nargs` of them given.
@@ -7254,6 +7285,9 @@ struct Lower<'a> {
     /// The block a helper's raise goes to (see [`Self::leave`]), made on
     /// first use.
     catch: Option<Block>,
+    /// The block every [`Self::leave`] jumps to with its status, made on
+    /// first use.
+    leave_tail: Option<Block>,
 }
 
 /// An exit block of [`Lower::exit_with`], filled after the body from the
@@ -7433,6 +7467,7 @@ impl<'a> Lower<'a> {
             held: Vec::new(),
             mod_caches: Vec::new(),
             catch: None,
+            leave_tail: None,
         }
     }
 
@@ -7777,6 +7812,24 @@ impl<'a> Lower<'a> {
     /// Return a helper's status `r` from the native code, a raise first
     /// trying the running activation's handlers (see [`h_catch`]).
     fn leave(&mut self, r: Value) {
+        // (One shared tail: a test, a branch and a return per helper call
+        // cost more compile memory than a jump.)
+        let tail = *self.leave_tail.get_or_insert_with(|| {
+            let tail = self.b.create_block();
+            self.b.append_block_param(tail, types::I32);
+            tail
+        });
+        self.b.ins().jump(tail, &[r.into()]);
+    }
+
+    /// Fill the shared tail of [`Self::leave`]: a raise tries the running
+    /// activation's handlers, any other status returns.
+    fn leave_tail(&mut self) {
+        let Some(tail) = self.leave_tail else {
+            return;
+        };
+        self.b.switch_to_block(tail);
+        let r = self.b.block_params(tail)[0];
         let catch = *self.catch.get_or_insert_with(|| self.b.create_block());
         let ret = self.b.create_block();
         let raised = self.b.ins().icmp_imm(IntCC::Equal, r, i64::from(RAISED));
@@ -7788,6 +7841,7 @@ impl<'a> Lower<'a> {
     /// Fill the side exits' blocks, and the catch block: a raise its
     /// handler caught goes on through the dispatch (at the handler).
     fn side_exits(&mut self) {
+        self.leave_tail();
         if let Some(catch) = self.catch {
             self.b.switch_to_block(catch);
             let r = self
@@ -9570,7 +9624,11 @@ impl<'a> Lower<'a> {
     /// raise or switch, which leaves the whole stack on the frame's: only
     /// a stack with nothing virtual below the operands tries it.
     fn binop_declined(&mut self, pc: usize, slot: usize, r: Value, out: Block, ok: Block) {
-        if !self.vs.iter().all(|i| matches!(i, Item::Mem(_))) {
+        // (A lean body tries the method only at a site that has run on
+        // instances: elsewhere the core loop runs it.)
+        if !self.vs.iter().all(|i| matches!(i, Item::Mem(_)))
+            || (self.cold && !crate::instance_site(self.ext, pc))
+        {
             self.b.ins().brif(r, out, &[], ok, &[]);
             return;
         }
@@ -11223,15 +11281,45 @@ impl<'a> Lower<'a> {
                 .expect("returns");
             return self.called(pc, argc, r);
         }
-        // A Python function's call tries a direct call first, unless its
-        // site found the callee without a native body and has retries
-        // left to count down.
         let site: Box<DirectSite> = Box::default();
         let site_at = self
             .b
             .ins()
             .iconst(self.ptr, std::ptr::from_ref::<DirectSite>(&site) as i64);
         self.direct_sites.push(site);
+        let r = if self.cold {
+            // A lean body makes [`Self::call_choice`]'s choice in a helper.
+            let pc_argc = self
+                .b
+                .ins()
+                .iconst(types::I64, (pc as i64) | ((argc as i64) << 32));
+            self.call(
+                h_call_site as *const () as usize,
+                &[self.st, code, ext, pc_argc, len, site_at],
+                true,
+            )
+            .expect("returns")
+        } else {
+            self.call_choice(pc, argc, code, ext, pcv, len, site_at)
+        };
+        self.called(pc, argc, r)
+    }
+
+    /// [`Self::call_op`]'s choice in line: a Python function's call tries
+    /// a direct call first, unless its site found the callee without a
+    /// native body and has retries left to count down (as
+    /// [`h_call_site`]). The helper's status.
+    #[allow(clippy::too_many_arguments)]
+    fn call_choice(
+        &mut self,
+        pc: usize,
+        argc: usize,
+        code: Value,
+        ext: Value,
+        pcv: Value,
+        len: Value,
+        site_at: Value,
+    ) -> Value {
         let callee = self.slot_addr(self.depth - argc - 2);
         let tag = self.tag_at(callee);
         let is_fn = self
@@ -11280,8 +11368,7 @@ impl<'a> Lower<'a> {
             .expect("returns");
         self.b.ins().jump(called, &[r.into()]);
         self.b.switch_to_block(called);
-        let r = self.b.block_params(called)[0];
-        self.called(pc, argc, r)
+        self.b.block_params(called)[0]
     }
 
     /// The helper and its site for a `CALL` (of `argc` arguments) at `pc`

From 3c7605c2d7789ffccb530303efdf3068dc595aff Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sat, 10 Oct 2026 23:21:16 -0700
Subject: [PATCH 150/173] perf: hash the compiler's name sets with FxHash
 instead of SipHash

Scope analysis (read and free-name collection, the validator, the
inlined-use scan) builds many small HashSet<String>s per code object,
and the symbol table is an IndexMap<String, Binding>; all used the
standard library's keyed SipHash. A multiply-rotate hasher (rustc's
FxHasher) does the same job in a fraction of the instructions, and the
compiler never hashes enough attacker-chosen names for flooding to
matter.

Measured (instructions retired, macOS, pinned hash seed):
- compile() of the frozen/non-frozen dataclass __create_fn__ sources:
  1.190M -> 1.089M per compile
- compile() of namedtuple's __new__ lambda: 113k -> 103k
- class_creation probe (type, dataclass, Enum, namedtuple per
  iteration): 4.97M -> 4.85M per iteration
---
 crates/weavepy-compiler/src/flowgraph.rs |  12 +-
 crates/weavepy-compiler/src/fxhash.rs    |  81 ++++++++++
 crates/weavepy-compiler/src/lib.rs       | 191 ++++++++++++-----------
 crates/weavepy-compiler/src/mangle.rs    |   6 +-
 crates/weavepy-compiler/src/validate.rs  |  24 +--
 5 files changed, 198 insertions(+), 116 deletions(-)
 create mode 100644 crates/weavepy-compiler/src/fxhash.rs

diff --git a/crates/weavepy-compiler/src/flowgraph.rs b/crates/weavepy-compiler/src/flowgraph.rs
index a13d701d..19f9b458 100644
--- a/crates/weavepy-compiler/src/flowgraph.rs
+++ b/crates/weavepy-compiler/src/flowgraph.rs
@@ -398,19 +398,19 @@ fn is_unary_not(i: &CfgInstr) -> bool {
 pub(crate) struct BuildInput<'a> {
     /// Unconditional jumps CPython's codegen emits as the plain `JUMP`
     /// pseudo-op; every other one is `JUMP_NO_INTERRUPT`.
-    pub plain_jumps: &'a std::collections::HashSet<u32>,
+    pub plain_jumps: &'a crate::fxhash::HashSet<u32>,
     /// Unconditional jumps codegen flagged `JUMP_NO_INTERRUPT`.
-    pub no_interrupt_jumps: &'a std::collections::HashSet<u32>,
+    pub no_interrupt_jumps: &'a crate::fxhash::HashSet<u32>,
     /// `Nop`s standing in for a `SETUP_*` pseudo-op.
-    pub setup_nops: &'a std::collections::HashMap<u32, OpCode>,
+    pub setup_nops: &'a crate::fxhash::HashMap<u32, OpCode>,
     /// `Nop`s standing in for a `POP_BLOCK` pseudo-op.
-    pub popblock_nops: &'a std::collections::HashSet<u32>,
+    pub popblock_nops: &'a crate::fxhash::HashSet<u32>,
     /// Conditional jumps that, together with the `COPY 1; TO_BOOL`
     /// preceding them, stand for a `JUMP_IF_FALSE` / `JUMP_IF_TRUE`
     /// pseudo-op.
-    pub pseudo_cond_jumps: &'a std::collections::HashSet<u32>,
+    pub pseudo_cond_jumps: &'a crate::fxhash::HashSet<u32>,
     /// Conditional jumps whose `arg` is a *backward* distance.
-    pub backward_conds: &'a std::collections::HashSet<u32>,
+    pub backward_conds: &'a crate::fxhash::HashSet<u32>,
     /// The code unit carries the PEP 479 `SETUP_CLEANUP` wrap (its
     /// range is the last exception-table entry).
     pub stopiteration_wrap: bool,
diff --git a/crates/weavepy-compiler/src/fxhash.rs b/crates/weavepy-compiler/src/fxhash.rs
new file mode 100644
index 00000000..a5fc7c28
--- /dev/null
+++ b/crates/weavepy-compiler/src/fxhash.rs
@@ -0,0 +1,81 @@
+//! A fast, unkeyed hasher for the compiler's name sets and binding maps.
+//!
+//! Scope analysis builds many small sets of identifiers per code object.
+//! The standard library's keyed SipHash costs several times more per short
+//! string than this multiply-rotate hash (rustc's `FxHasher`), and the
+//! compiler never hashes attacker-chosen keys at a scale where hash
+//! flooding matters: a source's names are bounded by its own size.
+
+use std::hash::{BuildHasherDefault, Hasher};
+
+/// rustc's `FxHasher`: one rotate, xor and multiply per word.
+#[derive(Default, Clone, Copy)]
+pub(crate) struct FxHasher {
+    hash: u64,
+}
+
+const SEED: u64 = 0x51_7c_c1_b7_27_22_0a_95;
+
+impl FxHasher {
+    #[inline]
+    fn add(&mut self, word: u64) {
+        self.hash = (self.hash.rotate_left(5) ^ word).wrapping_mul(SEED);
+    }
+}
+
+impl Hasher for FxHasher {
+    #[inline]
+    fn write(&mut self, mut bytes: &[u8]) {
+        while let Some((head, rest)) = bytes.split_first_chunk::<8>() {
+            self.add(u64::from_le_bytes(*head));
+            bytes = rest;
+        }
+        if let Some((head, rest)) = bytes.split_first_chunk::<4>() {
+            self.add(u64::from(u32::from_le_bytes(*head)));
+            bytes = rest;
+        }
+        if let Some((head, rest)) = bytes.split_first_chunk::<2>() {
+            self.add(u64::from(u16::from_le_bytes(*head)));
+            bytes = rest;
+        }
+        if let Some(&b) = bytes.first() {
+            self.add(u64::from(b));
+        }
+    }
+
+    #[inline]
+    fn write_u8(&mut self, i: u8) {
+        self.add(u64::from(i));
+    }
+
+    #[inline]
+    fn write_u32(&mut self, i: u32) {
+        self.add(u64::from(i));
+    }
+
+    #[inline]
+    fn write_u64(&mut self, i: u64) {
+        self.add(i);
+    }
+
+    #[inline]
+    fn write_usize(&mut self, i: usize) {
+        self.add(i as u64);
+    }
+
+    #[inline]
+    fn finish(&self) -> u64 {
+        self.hash
+    }
+}
+
+pub(crate) type FxBuild = BuildHasherDefault<FxHasher>;
+
+/// A `HashSet` over [`FxHasher`]; build one with `HashSet::default()`.
+pub(crate) type HashSet<T> = std::collections::HashSet<T, FxBuild>;
+
+/// A `HashMap` over [`FxHasher`]; build one with `HashMap::default()`.
+pub(crate) type HashMap<K, V> = std::collections::HashMap<K, V, FxBuild>;
+
+/// An `IndexMap` over [`FxHasher`]; build one with `IndexMap::default()`.
+pub(crate) type IndexMap<K, V> = indexmap::IndexMap<K, V, FxBuild>;
diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs
index a2dc3920..556a4f8d 100644
--- a/crates/weavepy-compiler/src/lib.rs
+++ b/crates/weavepy-compiler/src/lib.rs
@@ -20,12 +20,12 @@
 //!   peephole optimizer and adaptive specialization produce different
 //!   shapes that we deliberately don't reproduce.
 
+use fxhash::HashSet;
 use std::borrow::Cow;
-use std::collections::HashSet;
 use std::rc::Rc;
 
 use bytecode::intrinsic;
-use indexmap::IndexMap;
+use fxhash::IndexMap;
 use thiserror::Error;
 use weavepy_parser::ast::{
     Arguments as AstArguments, BinOp, BoolOp, CmpOp, Comprehension, Constant as AstConstant,
@@ -38,6 +38,7 @@ pub mod bytecode;
 mod cache_snapshot;
 pub mod cpython_code;
 mod flowgraph;
+mod fxhash;
 mod intern;
 mod mangle;
 mod name;
@@ -1798,7 +1799,7 @@ fn collect_module_imports(body: &[Stmt]) -> HashSet<String> {
             }
         }
     }
-    let mut out = HashSet::new();
+    let mut out = HashSet::default();
     walk(body, &mut out);
     out
 }
@@ -2447,7 +2448,7 @@ struct Compiler {
     /// `Nop`s standing in for a located `SETUP_*` pseudo-op (the `try:`
     /// line's NOP is really `SETUP_FINALLY`). The flowgraph keeps them
     /// as block-push instructions until `convert_pseudo_ops`.
-    setup_nops: std::collections::HashMap<u32, OpCode>,
+    setup_nops: fxhash::HashMap<u32, OpCode>,
     /// Suppress-path exit jumps parked by
     /// [`Self::emit_with_except_finish`] for its caller to patch.
     with_exit_jumps: Vec<u32>,
@@ -2820,10 +2821,10 @@ impl Compiler {
             name_index: intern::InternIndex::default(),
             kind,
             comp_kind: None,
-            bindings: IndexMap::new(),
-            explicit_globals: HashSet::new(),
-            comp_cell_only: HashSet::new(),
-            class_transparent_frees: HashSet::new(),
+            bindings: IndexMap::default(),
+            explicit_globals: HashSet::default(),
+            comp_cell_only: HashSet::default(),
+            class_transparent_frees: HashSet::default(),
             free_order: Vec::new(),
             loop_stack: Vec::new(),
             finally_stack: Vec::new(),
@@ -2834,13 +2835,13 @@ impl Compiler {
             line_pinned: None,
             pinned_colspan: ColSpan::default(),
             current_span: (0, 0),
-            no_interrupt_jumps: HashSet::new(),
-            plain_jumps: HashSet::new(),
-            setup_nops: std::collections::HashMap::new(),
+            no_interrupt_jumps: HashSet::default(),
+            plain_jumps: HashSet::default(),
+            setup_nops: fxhash::HashMap::default(),
             with_exit_jumps: Vec::new(),
-            popblock_nops: HashSet::new(),
-            pseudo_cond_jumps: HashSet::new(),
-            backward_conds: HashSet::new(),
+            popblock_nops: HashSet::default(),
+            pseudo_cond_jumps: HashSet::default(),
+            backward_conds: HashSet::default(),
             stopiteration_wrap: true,
             inline_comp: 0,
             module_comp_plain_targets: Vec::new(),
@@ -2864,7 +2865,7 @@ impl Compiler {
             pep695_defaults: None,
             lazy_class_ctx: None,
             pending_lazy_class_ctx: None,
-            class_assigned: HashSet::new(),
+            class_assigned: HashSet::default(),
             deferred_annotations: Vec::new(),
             next_cond_annotation_index: 0,
             in_conditional_block: 0,
@@ -3752,7 +3753,7 @@ impl Compiler {
     fn analyze_scope_module(&mut self, module: &Module) {
         // At module scope every assigned name is a global (CPython
         // does the same — locals at module scope ARE the globals).
-        let mut assigned = HashSet::new();
+        let mut assigned = HashSet::default();
         for s in &module.body {
             collect_assigned(s, &mut assigned);
             collect_walrus_stmt(s, &mut assigned);
@@ -3766,7 +3767,7 @@ impl Compiler {
         // accesses compile to the *_GLOBAL opcodes instead of *_NAME.
         // Observable when `exec` runs with distinct globals/locals
         // mappings (test_scope.testGlobalInParallelNestedFunctions).
-        let mut explicit = HashSet::new();
+        let mut explicit = HashSet::default();
         for s in &module.body {
             collect_global_decls_deep(s, &mut explicit);
         }
@@ -3786,7 +3787,7 @@ impl Compiler {
             }
         }
         self.explicit_globals = explicit;
-        let mut comp_cells = HashSet::new();
+        let mut comp_cells = HashSet::default();
         for s in &module.body {
             self.collect_comp_cells_stmt(s, &mut comp_cells);
         }
@@ -3829,7 +3830,7 @@ impl Compiler {
         collect: impl Fn(&FreeScan, &mut HashSet<String>),
     ) -> HashSet<String> {
         let scan = self.free_scan(inline_comps);
-        let mut needed = HashSet::new();
+        let mut needed = HashSet::default();
         collect(&scan, &mut needed);
         needed
     }
@@ -3881,9 +3882,9 @@ impl Compiler {
         for p in params {
             self.bindings.insert(p.clone(), Binding::Local);
         }
-        let mut globals = HashSet::new();
-        let mut nonlocals = HashSet::new();
-        let mut assigned = HashSet::new();
+        let mut globals = HashSet::default();
+        let mut nonlocals = HashSet::default();
+        let mut assigned = HashSet::default();
         for s in body {
             collect_decls(s, &mut globals, &mut nonlocals, &mut assigned);
             // Walrus targets bind in this scope too (PEP 572) but live inside
@@ -3913,7 +3914,7 @@ impl Compiler {
             self.bindings.entry(n).or_insert(Binding::Local);
         }
         // Names referenced by directly-emitted bytecode in this scope.
-        let mut reads = HashSet::new();
+        let mut reads = HashSet::default();
         for s in body {
             collect_reads_stmt_fn(s, &mut reads);
         }
@@ -3983,7 +3984,7 @@ impl Compiler {
             }
         }
         if self.lazy_class_ctx.is_none() {
-            let mut comp_cells = HashSet::new();
+            let mut comp_cells = HashSet::default();
             for s in body {
                 self.collect_comp_cells_stmt(s, &mut comp_cells);
             }
@@ -3993,7 +3994,7 @@ impl Compiler {
             // (`return b` next to `[(lambda: b) for b in xs]` is a
             // LOAD_GLOBAL, test_listcomps
             // `test_global_outside_cellvar_inside_plus_freevar`).
-            let mut used = HashSet::new();
+            let mut used = HashSet::default();
             for s in body {
                 for_each_scope_expr(s, &mut |e| {
                     self.collect_uses_through_inlined(e, &mut Vec::new(), &mut used)
@@ -5613,7 +5614,7 @@ impl Compiler {
         // A class's own type parameters (and the header expressions
         // that mention them) mangle against the class's name.
         let mut type_params = type_params.clone();
-        let mut own_params: HashSet<String> = HashSet::new();
+        let mut own_params: HashSet<String> = HashSet::default();
         let mangle_header = is_class && !display.trim_start_matches('_').is_empty();
         if mangle_header {
             // Source spellings: `mangle_only_in_expr` renames exactly
@@ -6421,9 +6422,9 @@ impl Compiler {
         // never satisfies a nonlocal, and a class-body Local is a class
         // attribute, not a nonlocal binding target.
         {
-            let mut ng = HashSet::new();
-            let mut nl = HashSet::new();
-            let mut na = HashSet::new();
+            let mut ng = HashSet::default();
+            let mut nl = HashSet::default();
+            let mut na = HashSet::default();
             for s in body {
                 collect_decls(s, &mut ng, &mut nl, &mut na);
             }
@@ -7144,7 +7145,7 @@ impl Compiler {
                 class_body: false,
                 class_binds: None,
             };
-            let mut needed = HashSet::new();
+            let mut needed = HashSet::default();
             for s in body {
                 collect_inner_free(s, &self.bindings, &mut needed, &scan);
             }
@@ -7212,7 +7213,7 @@ impl Compiler {
                 .insert("__conditional_annotations__".to_owned(), Binding::Cell);
         }
 
-        let mut assigned = HashSet::new();
+        let mut assigned = HashSet::default();
         for s in body {
             collect_assigned(s, &mut assigned);
         }
@@ -7226,9 +7227,9 @@ impl Compiler {
         // nested `def X`/`class X` gets a bare qualname (see
         // `compute_child_qualname`).
         {
-            let mut globals = HashSet::new();
-            let mut nonlocals = HashSet::new();
-            let mut decl_assigned = HashSet::new();
+            let mut globals = HashSet::default();
+            let mut nonlocals = HashSet::default();
+            let mut decl_assigned = HashSet::default();
             for s in body {
                 collect_decls(s, &mut globals, &mut nonlocals, &mut decl_assigned);
             }
@@ -7269,7 +7270,7 @@ impl Compiler {
 
         // Resolve outer-scope free vars for names read by the body that
         // aren't bound locally.
-        let mut reads = HashSet::new();
+        let mut reads = HashSet::default();
         for s in body {
             collect_reads_stmt(s, &mut reads);
         }
@@ -7350,7 +7351,7 @@ impl Compiler {
         // `__classdict__` cells (`dictbytype` first, the implicit
         // cells appended by `compiler_enter_scope`).
         {
-            let mut comp_cells = HashSet::new();
+            let mut comp_cells = HashSet::default();
             for s in body {
                 inner.collect_comp_cells_stmt(s, &mut comp_cells);
             }
@@ -7536,7 +7537,7 @@ impl Compiler {
         // the class scope (nested functions included; nested classes
         // collect their own), regardless of parameter names.
         {
-            let mut attrs: HashSet<String> = HashSet::new();
+            let mut attrs: HashSet<String> = HashSet::default();
             collect_self_attr_stores(unmangled_body, "self", &mut attrs);
             let mut attrs: Vec<String> = attrs.into_iter().collect();
             attrs.sort();
@@ -11216,7 +11217,7 @@ impl Compiler {
         value: Option<&Expr>,
         generators: &[Comprehension],
     ) -> HashSet<String> {
-        let mut needed = HashSet::new();
+        let mut needed = HashSet::default();
         collect_comp_scope_inner_free(
             elt,
             value,
@@ -11225,7 +11226,7 @@ impl Compiler {
             &mut needed,
             &self.free_scan(true),
         );
-        let mut targets = HashSet::new();
+        let mut targets = HashSet::default();
         for g in generators {
             collect_target_names(&g.target, &mut targets);
         }
@@ -11280,7 +11281,7 @@ impl Compiler {
         // `[x for [x for x in r][0] in l]` keeps the outer `x` as the
         // enclosing scope's variable rather than a hidden local
         // (test_listcomps `test_nested_2`).
-        let mut used = HashSet::new();
+        let mut used = HashSet::default();
         for (gi, g) in generators.iter().enumerate() {
             Self::collect_names_shallow(&g.target, &mut used);
             if gi > 0 {
@@ -11365,7 +11366,7 @@ impl Compiler {
         }
         let depth = bound.len();
         for g in generators {
-            let mut names = HashSet::new();
+            let mut names = HashSet::default();
             collect_target_names(&g.target, &mut names);
             bound.extend(names);
         }
@@ -11857,7 +11858,7 @@ impl Compiler {
         // an extra strong reference for the whole activation, and pandas'
         // chained-assignment detection (`getrefcount(self) <= 3` in
         // `DataFrame.__setitem__`) is calibrated against CPython counts.
-        let mut reads = HashSet::new();
+        let mut reads = HashSet::default();
         collect_reads_expr(elt, &mut reads);
         if let Some(v) = value {
             collect_reads_expr(v, &mut reads);
@@ -11880,7 +11881,7 @@ impl Compiler {
         // one — silently aliasing later closure reads. CPython's symtable binds
         // comprehension targets first for exactly this reason.
         for g in generators {
-            let mut assigned = HashSet::new();
+            let mut assigned = HashSet::default();
             collect_target_names(&g.target, &mut assigned);
             for n in assigned {
                 inner.bindings.insert(n, Binding::Local);
@@ -12014,7 +12015,7 @@ impl Compiler {
                 }
             }
             if inner.lazy_class_ctx.is_none() {
-                let mut comp_cells = HashSet::new();
+                let mut comp_cells = HashSet::default();
                 for (gi, g) in generators.iter().enumerate() {
                     if gi > 0 {
                         inner.collect_comp_cells_expr(&g.iter, &mut comp_cells);
@@ -12659,7 +12660,7 @@ fn check_comp_walrus_nest(
         // iterable (test_named_expressions' "Lambda expression" /
         // "Nested comprehension body" cases).
         reject_walrus_in_iterable(&g.iter)?;
-        let mut names = HashSet::new();
+        let mut names = HashSet::default();
         collect_target_names(&g.target, &mut names);
         for scope in stack.iter() {
             for n in &names {
@@ -13081,7 +13082,7 @@ fn collect_pep695_header_reads(
     out: &mut HashSet<String>,
     scan: &FreeScan,
 ) {
-    let mut reads = HashSet::new();
+    let mut reads = HashSet::default();
     let mut read = |e: &Expr| collect_class_visible_reads(e, outer_bindings, &mut reads, scan);
     let (type_params, fn_parts) = match &stmt.kind {
         StmtKind::FunctionDef {
@@ -13353,7 +13354,7 @@ fn collect_inner_free(
     // enclosing `T = …` must not become a cell for a nested
     // `def f[T](x: T)`).
     if let Some(type_params) = generic_type_params(stmt) {
-        let mut inner = HashSet::new();
+        let mut inner = HashSet::default();
         collect_pep695_header_reads(stmt, outer_bindings, &mut inner, scan);
         collect_inner_free_impl(stmt, outer_bindings, &mut inner, scan);
         remove_type_param_names(stmt, type_params, &mut inner);
@@ -13490,7 +13491,7 @@ fn collect_inner_free_impl(
             // The body is its own scope, no longer directly in a class
             // body.
             let scan = &scan.nested();
-            let mut inner_locals: HashSet<String> = HashSet::new();
+            let mut inner_locals: HashSet<String> = HashSet::default();
             for a in &args.posonlyargs {
                 inner_locals.insert(a.name.clone());
             }
@@ -13506,9 +13507,9 @@ fn collect_inner_free_impl(
             if let Some(kw) = &args.kwarg {
                 inner_locals.insert(kw.name.clone());
             }
-            let mut inner_globals = HashSet::new();
-            let mut inner_nonlocals = HashSet::new();
-            let mut inner_assigned = HashSet::new();
+            let mut inner_globals = HashSet::default();
+            let mut inner_nonlocals = HashSet::default();
+            let mut inner_assigned = HashSet::default();
             for s in body {
                 collect_decls(
                     s,
@@ -13527,7 +13528,7 @@ fn collect_inner_free_impl(
             }
             // Reads inside the inner that aren't locals there →
             // candidates for promotion.
-            let mut inner_reads = HashSet::new();
+            let mut inner_reads = HashSet::default();
             for s in body {
                 collect_reads_stmt_fn(s, &mut inner_reads);
             }
@@ -13554,7 +13555,7 @@ fn collect_inner_free_impl(
             // the child's cell, not ours (`def _tp_cache(func): def
             // decorator(func): def inner(): func` must not promote the
             // outer `func`). `nonlocal` in the child reaches through.
-            let mut deeper = HashSet::new();
+            let mut deeper = HashSet::default();
             for s in body {
                 collect_inner_free(s, outer_bindings, &mut deeper, scan);
             }
@@ -13601,26 +13602,26 @@ fn collect_inner_free_impl(
                 mangled_body = b;
                 &mangled_body
             };
-            let mut class_assigned = HashSet::new();
-            let mut class_globals = HashSet::new();
-            let mut class_nonlocals = HashSet::new();
+            let mut class_assigned = HashSet::default();
+            let mut class_globals = HashSet::default();
+            let mut class_nonlocals = HashSet::default();
             for s in body {
                 collect_assigned(s, &mut class_assigned);
                 collect_decls(
                     s,
                     &mut class_globals,
                     &mut class_nonlocals,
-                    &mut HashSet::new(),
+                    &mut HashSet::default(),
                 );
             }
             // Names referenced *anywhere* in the class body (including
             // method bodies) that aren't bound inside the class are
             // candidates for outer-scope free promotion.
-            let mut class_reads = HashSet::new();
+            let mut class_reads = HashSet::default();
             for s in body {
                 collect_reads_stmt(s, &mut class_reads);
             }
-            let mut from_body = HashSet::new();
+            let mut from_body = HashSet::default();
             for r in class_reads {
                 if !class_assigned.contains(&r) {
                     from_body.insert(r);
@@ -13780,7 +13781,7 @@ fn collect_inner_free_impl(
                 // (GLOBAL_IMPLICIT), never as a free variable of the
                 // enclosing function (`class D: C: C = field()` inside
                 // a `def` where `C` is a local).
-                let mut reads = HashSet::new();
+                let mut reads = HashSet::default();
                 collect_reads_expr(annotation, &mut reads);
                 for r in reads {
                     if !scan.class_binds.as_ref().is_some_and(|b| b.contains(&r)) {
@@ -13830,9 +13831,9 @@ fn collect_inner_free_impl(
 /// `True` when a method body references `super` or `__class__` so the
 /// compiler knows to capture the class's `__class__` cell.
 fn method_references_class(body: &[Stmt]) -> bool {
-    let mut globals = HashSet::new();
-    let mut nonlocals = HashSet::new();
-    let mut assigned = HashSet::new();
+    let mut globals = HashSet::default();
+    let mut nonlocals = HashSet::default();
+    let mut assigned = HashSet::default();
     for s in body {
         collect_decls(s, &mut globals, &mut nonlocals, &mut assigned);
         collect_walrus_stmt(s, &mut assigned);
@@ -13927,7 +13928,7 @@ fn body_reads_super_or_class(body: &[Stmt]) -> bool {
         }
         any_class(stmts)
     };
-    let mut reads = HashSet::new();
+    let mut reads = HashSet::default();
     if has_nested_class(body) {
         // Only clone when there is a class body to blank out.
         let mut stripped = body.to_vec();
@@ -14694,7 +14695,7 @@ fn collect_class_visible_reads(
     out: &mut HashSet<String>,
     scan: &FreeScan,
 ) {
-    let mut all = HashSet::new();
+    let mut all = HashSet::default();
     collect_reads_expr(expr, &mut all);
     if let Some(binds) = &scan.class_binds {
         all.retain(|n| !binds.contains(n));
@@ -14750,7 +14751,7 @@ fn collect_inlined_comp_inner_free(
     if let Some(first) = generators.first() {
         collect_inner_free_expr(&first.iter, outer_bindings, out, scan);
     }
-    let mut inner = HashSet::new();
+    let mut inner = HashSet::default();
     collect_comp_scope_inner_free(elt, value, generators, outer_bindings, &mut inner, scan);
     if scan.class_body {
         // The class namespace is invisible from inside the
@@ -14770,7 +14771,7 @@ fn collect_inlined_comp_inner_free(
         }
         collect_reads_expr(elt, &mut inner);
     }
-    let mut bound = HashSet::new();
+    let mut bound = HashSet::default();
     for g in generators {
         collect_target_names(&g.target, &mut bound);
     }
@@ -14821,7 +14822,7 @@ fn collect_inner_free_expr(
             collect_class_visible_reads(body, outer_bindings, out, scan);
         }
         ExprKind::Lambda { args, body } | ExprKind::TypeParamFn { args, body } => {
-            let mut inner_locals: HashSet<String> = HashSet::new();
+            let mut inner_locals: HashSet<String> = HashSet::default();
             for a in &args.posonlyargs {
                 inner_locals.insert(a.name.clone());
             }
@@ -14837,7 +14838,7 @@ fn collect_inner_free_expr(
             if let Some(kw) = &args.kwarg {
                 inner_locals.insert(kw.name.clone());
             }
-            let mut reads = HashSet::new();
+            let mut reads = HashSet::default();
             collect_reads_deep(body, &mut reads);
             for r in reads {
                 if !inner_locals.contains(&r) {
@@ -14871,11 +14872,11 @@ fn collect_inner_free_expr(
         ExprKind::ListComp { elt, generators }
         | ExprKind::SetComp { elt, generators }
         | ExprKind::GeneratorExp { elt, generators } => {
-            let mut inner_locals: HashSet<String> = HashSet::new();
+            let mut inner_locals: HashSet<String> = HashSet::default();
             for g in generators {
                 collect_target_names(&g.target, &mut inner_locals);
             }
-            let mut reads = HashSet::new();
+            let mut reads = HashSet::default();
             collect_reads_deep(elt, &mut reads);
             for (gi, g) in generators.iter().enumerate() {
                 // The outermost iterable evaluates in the *enclosing*
@@ -14907,11 +14908,11 @@ fn collect_inner_free_expr(
             value,
             generators,
         } => {
-            let mut inner_locals: HashSet<String> = HashSet::new();
+            let mut inner_locals: HashSet<String> = HashSet::default();
             for g in generators {
                 collect_target_names(&g.target, &mut inner_locals);
             }
-            let mut reads = HashSet::new();
+            let mut reads = HashSet::default();
             collect_reads_deep(key, &mut reads);
             collect_reads_deep(value, &mut reads);
             for (gi, g) in generators.iter().enumerate() {
@@ -15965,7 +15966,7 @@ fn class_level_reads_dunder(body: &[Stmt], name: &str) -> bool {
 /// enclosing function's `c`, while `class B: c: str` or `class B: x = 1;
 /// y = x` alone never does).
 fn class_body_propagating_reads(body: &[Stmt], out: &mut HashSet<String>) {
-    let mut inner = HashSet::new();
+    let mut inner = HashSet::default();
     for s in body {
         collect_reads_stmt_in(s, &mut inner, false);
     }
@@ -15984,16 +15985,16 @@ fn class_body_propagating_reads(body: &[Stmt], out: &mut HashSet<String>) {
     // when read at class level, and through `__classdict__` in a
     // class-visible annotation scope; only a nested scope's free read
     // of it looks past the class.
-    let mut class_bound = HashSet::new();
-    let mut class_globals = HashSet::new();
-    let mut class_nonlocals = HashSet::new();
+    let mut class_bound = HashSet::default();
+    let mut class_globals = HashSet::default();
+    let mut class_nonlocals = HashSet::default();
     for s in body {
         collect_assigned(s, &mut class_bound);
         collect_decls(
             s,
             &mut class_globals,
             &mut class_nonlocals,
-            &mut HashSet::new(),
+            &mut HashSet::default(),
         );
     }
     let shadowed: Vec<String> = inner
@@ -16005,7 +16006,7 @@ fn class_body_propagating_reads(body: &[Stmt], out: &mut HashSet<String>) {
         .cloned()
         .collect();
     if !shadowed.is_empty() {
-        let mut nested = HashSet::new();
+        let mut nested = HashSet::default();
         class_nested_scope_reads(body, &mut nested);
         for n in shadowed {
             if !nested.contains(&n) {
@@ -16031,7 +16032,7 @@ fn class_nested_scope_reads(body: &[Stmt], out: &mut HashSet<String>) {
                 {
                     expr_nested(d, out);
                 }
-                let mut inner = HashSet::new();
+                let mut inner = HashSet::default();
                 collect_reads_expr(body, &mut inner);
                 for a in args
                     .posonlyargs
@@ -16071,7 +16072,7 @@ fn class_nested_scope_reads(body: &[Stmt], out: &mut HashSet<String>) {
         if let Some(first) = generators.first() {
             expr_nested(&first.iter, out);
         }
-        let mut inner = HashSet::new();
+        let mut inner = HashSet::default();
         for (gi, g) in generators.iter().enumerate() {
             if gi > 0 {
                 collect_reads_expr(&g.iter, &mut inner);
@@ -16085,7 +16086,7 @@ fn class_nested_scope_reads(body: &[Stmt], out: &mut HashSet<String>) {
             collect_reads_expr(v, &mut inner);
         }
         collect_reads_expr(elt, &mut inner);
-        let mut bound = HashSet::new();
+        let mut bound = HashSet::default();
         for g in generators {
             collect_target_names(&g.target, &mut bound);
         }
@@ -16146,7 +16147,7 @@ fn class_nested_scope_reads(body: &[Stmt], out: &mut HashSet<String>) {
                     mangled_body = b;
                     &mangled_body
                 };
-                let mut through = HashSet::new();
+                let mut through = HashSet::default();
                 class_body_propagating_reads(body, &mut through);
                 for dunder in CLASS_DUNDERS {
                     through.remove(dunder);
@@ -16265,11 +16266,11 @@ fn class_nested_scope_reads(body: &[Stmt], out: &mut HashSet<String>) {
 /// reach up even when only written). Decorators, defaults and
 /// annotations are the enclosing scope's business.
 fn function_free_names(args: &AstArguments, body: &[Stmt]) -> HashSet<String> {
-    let mut nested_reads = HashSet::new();
+    let mut nested_reads = HashSet::default();
     for s in body {
         collect_reads_stmt_in(s, &mut nested_reads, true);
     }
-    let mut nested_locals: HashSet<String> = HashSet::new();
+    let mut nested_locals: HashSet<String> = HashSet::default();
     for a in args
         .posonlyargs
         .iter()
@@ -16280,9 +16281,9 @@ fn function_free_names(args: &AstArguments, body: &[Stmt]) -> HashSet<String> {
     {
         nested_locals.insert(a.name.clone());
     }
-    let mut nested_globals = HashSet::new();
-    let mut nested_nonlocals = HashSet::new();
-    let mut nested_assigned = HashSet::new();
+    let mut nested_globals = HashSet::default();
+    let mut nested_nonlocals = HashSet::default();
+    let mut nested_assigned = HashSet::default();
     for s in body {
         collect_decls(
             s,
@@ -16433,7 +16434,7 @@ fn class_body_reads_dunder(body: &[Stmt], name: &str) -> bool {
             }
         }
     }
-    let mut reads = HashSet::new();
+    let mut reads = HashSet::default();
     for s in body {
         stmt_reads(s, &mut reads, name);
     }
@@ -16445,7 +16446,7 @@ fn class_body_reads_dunder(body: &[Stmt], name: &str) -> bool {
 /// scope are skipped except for the parts evaluated in the enclosing
 /// scope (lambda defaults, a comprehension's outermost iterable).
 fn stmt_reads_dunder_at_class_level(stmt: &Stmt, name: &str) -> bool {
-    let mut all = HashSet::new();
+    let mut all = HashSet::default();
     collect_reads_stmt(stmt, &mut all);
     if !all.contains(name) {
         return false;
@@ -17139,7 +17140,7 @@ fn collect_reads_stmt_in(stmt: &Stmt, out: &mut HashSet<String>, fn_scope: bool)
     // See `collect_inner_free`: a generic statement's type parameters
     // are its hidden scope's locals, never reads of this scope.
     if let Some(type_params) = generic_type_params(stmt) {
-        let mut inner = HashSet::new();
+        let mut inner = HashSet::default();
         collect_reads_stmt_in_impl(stmt, &mut inner, fn_scope);
         remove_type_param_names(stmt, type_params, &mut inner);
         out.extend(inner);
@@ -17271,7 +17272,7 @@ fn collect_reads_stmt_in_impl(stmt: &Stmt, out: &mut HashSet<String>, fn_scope:
         StmtKind::TypeAlias {
             type_params, value, ..
         } => {
-            let mut nested = HashSet::new();
+            let mut nested = HashSet::default();
             collect_reads_expr(value, &mut nested);
             for tp in type_params {
                 if let TypeParamKind::TypeVar { bound: Some(b) } = &tp.kind {
@@ -17458,7 +17459,7 @@ fn collect_comp_reads(
     if let Some(first) = generators.first() {
         collect_reads_expr(&first.iter, out);
     }
-    let mut inner = HashSet::new();
+    let mut inner = HashSet::default();
     for (gi, g) in generators.iter().enumerate() {
         if gi > 0 {
             collect_reads_expr(&g.iter, &mut inner);
@@ -17474,7 +17475,7 @@ fn collect_comp_reads(
         collect_reads_expr(v, &mut inner);
     }
     collect_reads_expr(elt, &mut inner);
-    let mut bound = HashSet::new();
+    let mut bound = HashSet::default();
     for g in generators {
         collect_target_names(&g.target, &mut bound);
     }
@@ -17554,7 +17555,7 @@ fn collect_reads_expr(expr: &Expr, out: &mut HashSet<String>) {
             }
             // The body is its own scope: what it reads reaches us only
             // when the lambda doesn't bind it itself (its parameters).
-            let mut inner = HashSet::new();
+            let mut inner = HashSet::default();
             collect_reads_expr(body, &mut inner);
             for a in args
                 .posonlyargs
diff --git a/crates/weavepy-compiler/src/mangle.rs b/crates/weavepy-compiler/src/mangle.rs
index acdfa7fe..bfbdd63b 100644
--- a/crates/weavepy-compiler/src/mangle.rs
+++ b/crates/weavepy-compiler/src/mangle.rs
@@ -15,7 +15,7 @@
 //! are mangled against their own (innermost) class name when their own
 //! body is compiled.
 
-use std::collections::HashSet;
+use crate::fxhash::HashSet;
 
 use weavepy_parser::ast::{
     Arguments, Comprehension, ExceptHandler, Expr, ExprKind, MatchCase, Pattern, PatternKind, Stmt,
@@ -45,7 +45,7 @@ pub(crate) fn mangle_class_body(class_name: &str, body: &mut [Stmt]) {
     }
     let m = Mangler {
         prefix: format!("_{stripped}"),
-        skip: HashSet::new(),
+        skip: HashSet::default(),
         only: None,
     };
     for s in body {
@@ -82,7 +82,7 @@ pub(crate) fn mangle_only_in_expr(class_name: &str, names: &HashSet<String>, exp
     }
     let m = Mangler {
         prefix: format!("_{stripped}"),
-        skip: HashSet::new(),
+        skip: HashSet::default(),
         only: Some(names.clone()),
     };
     m.expr(expr);
diff --git a/crates/weavepy-compiler/src/validate.rs b/crates/weavepy-compiler/src/validate.rs
index 70482ecb..d6a50a6c 100644
--- a/crates/weavepy-compiler/src/validate.rs
+++ b/crates/weavepy-compiler/src/validate.rs
@@ -68,17 +68,17 @@ struct Scope {
     /// for functions; the parameter names themselves for
     /// [`ScopeKind::TypeParams`]. Consulted when resolving `nonlocal`
     /// declarations from nested scopes.
-    bound: std::collections::HashSet<String>,
+    bound: crate::fxhash::HashSet<String>,
     /// Names *read* so far, in source order (CPython's `USE` flag) —
     /// a later `global`/`nonlocal` for one of these is "used prior to
     /// … declaration".
-    used: std::collections::HashSet<String>,
+    used: crate::fxhash::HashSet<String>,
     /// Names assigned/deleted/bound so far (`DEF_LOCAL`) — a later
     /// declaration is "assigned to before … declaration".
-    assigned: std::collections::HashSet<String>,
+    assigned: crate::fxhash::HashSet<String>,
     /// Names annotated so far (`DEF_ANNOT`) — can never be declared
     /// global/nonlocal in this scope, before *or* after.
-    annotated: std::collections::HashSet<String>,
+    annotated: crate::fxhash::HashSet<String>,
 }
 
 impl Scope {
@@ -87,10 +87,10 @@ impl Scope {
             kind,
             params: Vec::new(),
             directives: Vec::new(),
-            bound: std::collections::HashSet::new(),
-            used: std::collections::HashSet::new(),
-            assigned: std::collections::HashSet::new(),
-            annotated: std::collections::HashSet::new(),
+            bound: crate::fxhash::HashSet::default(),
+            used: crate::fxhash::HashSet::default(),
+            assigned: crate::fxhash::HashSet::default(),
+            annotated: crate::fxhash::HashSet::default(),
         }
     }
 
@@ -281,12 +281,12 @@ impl Validator<'_> {
         // Body-level bindings (assignments, defs, imports, walrus
         // targets) participate in `nonlocal` resolution from nested
         // scopes.
-        let mut bound: std::collections::HashSet<String> =
+        let mut bound: crate::fxhash::HashSet<String> =
             params.iter().map(|(n, _)| (*n).to_owned()).collect();
         {
-            let mut globals = std::collections::HashSet::new();
-            let mut nonlocals = std::collections::HashSet::new();
-            let mut assigned = std::collections::HashSet::new();
+            let mut globals = crate::fxhash::HashSet::default();
+            let mut nonlocals = crate::fxhash::HashSet::default();
+            let mut assigned = crate::fxhash::HashSet::default();
             for s in body {
                 crate::collect_decls(s, &mut globals, &mut nonlocals, &mut assigned);
                 crate::collect_walrus_stmt(s, &mut assigned);

From bb39959ea1bbafdb52e626bba7b397334d3c59cc Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 00:36:59 -0700
Subject: [PATCH 151/173] perf: skip the String allocation when a scope scan
 sees a known name

The compiler's scope scans (read collection, declared names, the
validator's used/assigned sets) inserted `name.clone()` into a
HashSet<String> for every occurrence, allocating and freeing a String
each time a name repeated (`self`, parameters). A small NameSet helper
looks the borrowed name up first and allocates only when it is new.

Measured (instructions retired, macOS, pinned hash seed): compile() of
the dataclass __create_fn__ sources, 1.089M -> 1.052M per compile.
---
 crates/weavepy-compiler/src/fxhash.rs   | 16 ++++++
 crates/weavepy-compiler/src/lib.rs      | 74 ++++++++++++-------------
 crates/weavepy-compiler/src/validate.rs |  7 ++-
 3 files changed, 57 insertions(+), 40 deletions(-)

diff --git a/crates/weavepy-compiler/src/fxhash.rs b/crates/weavepy-compiler/src/fxhash.rs
index a5fc7c28..af93bcaf 100644
--- a/crates/weavepy-compiler/src/fxhash.rs
+++ b/crates/weavepy-compiler/src/fxhash.rs
@@ -74,6 +74,22 @@ pub(crate) type FxBuild = BuildHasherDefault<FxHasher>;
 /// A `HashSet` over [`FxHasher`]; build one with `HashSet::default()`.
 pub(crate) type HashSet<T> = std::collections::HashSet<T, FxBuild>;
 
+/// Adding a borrowed name to a set of owned names, allocating only when
+/// the name is new: scope scans see the same few names (`self`, the
+/// parameters) over and over.
+pub(crate) trait NameSet {
+    fn add_name(&mut self, name: &str);
+}
+
+impl NameSet for HashSet<String> {
+    #[inline]
+    fn add_name(&mut self, name: &str) {
+        if !self.contains(name) {
+            self.insert(name.to_owned());
+        }
+    }
+}
+
 /// A `HashMap` over [`FxHasher`]; build one with `HashMap::default()`.
 pub(crate) type HashMap<K, V> = std::collections::HashMap<K, V, FxBuild>;
 
diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs
index 556a4f8d..3d839c1a 100644
--- a/crates/weavepy-compiler/src/lib.rs
+++ b/crates/weavepy-compiler/src/lib.rs
@@ -20,7 +20,7 @@
 //!   peephole optimizer and adaptive specialization produce different
 //!   shapes that we deliberately don't reproduce.
 
-use fxhash::HashSet;
+use fxhash::{HashSet, NameSet};
 use std::borrow::Cow;
 use std::rc::Rc;
 
@@ -1751,12 +1751,12 @@ fn collect_module_imports(body: &[Stmt]) -> HashSet<String> {
                     for a in aliases {
                         match &a.asname {
                             Some(n) => {
-                                out.insert(n.clone());
+                                out.add_name(n);
                             }
                             // `import a.b.c` binds the first segment.
                             None => {
                                 let first = a.name.split('.').next().unwrap_or(&a.name);
-                                out.insert(first.to_owned());
+                                out.add_name(first);
                             }
                         }
                     }
@@ -4002,7 +4002,7 @@ impl Compiler {
             }
             for name in &comp_cells {
                 if used.contains(name) && !self.bindings.contains_key(name) {
-                    self.comp_cell_only.insert(name.clone());
+                    self.comp_cell_only.add_name(name);
                 }
             }
             self.register_comp_cells(comp_cells);
@@ -7336,7 +7336,7 @@ impl Compiler {
                     // the class scope (PEP 227) and still reach the
                     // enclosing function's `y` — forward the cell
                     // without touching the class-level binding.
-                    inner.class_transparent_frees.insert(name.clone());
+                    inner.class_transparent_frees.add_name(name);
                 } else {
                     inner
                         .bindings
@@ -11329,7 +11329,7 @@ impl Compiler {
         let (kind, elt, value, generators) = match &expr.kind {
             ExprKind::Name(n) => {
                 if !bound.contains(n) {
-                    out.insert(n.clone());
+                    out.add_name(n);
                 }
                 return;
             }
@@ -11394,7 +11394,7 @@ impl Compiler {
     fn collect_names_shallow(expr: &Expr, out: &mut HashSet<String>) {
         match &expr.kind {
             ExprKind::Name(n) => {
-                out.insert(n.clone());
+                out.add_name(n);
             }
             ExprKind::Lambda { args, .. } | ExprKind::TypeParamFn { args, .. } => {
                 for d in &args.defaults {
@@ -12531,7 +12531,7 @@ fn check_comp_walrus_nest(
                         ));
                     }
                     for scope in stack.iter_mut() {
-                        scope.walrus_targets.insert(n.clone());
+                        scope.walrus_targets.add_name(n);
                     }
                 }
                 visit(value, in_class_body, stack)
@@ -13493,19 +13493,19 @@ fn collect_inner_free_impl(
             let scan = &scan.nested();
             let mut inner_locals: HashSet<String> = HashSet::default();
             for a in &args.posonlyargs {
-                inner_locals.insert(a.name.clone());
+                inner_locals.add_name(&a.name);
             }
             for a in &args.args {
-                inner_locals.insert(a.name.clone());
+                inner_locals.add_name(&a.name);
             }
             if let Some(va) = &args.vararg {
-                inner_locals.insert(va.name.clone());
+                inner_locals.add_name(&va.name);
             }
             for a in &args.kwonlyargs {
-                inner_locals.insert(a.name.clone());
+                inner_locals.add_name(&a.name);
             }
             if let Some(kw) = &args.kwarg {
-                inner_locals.insert(kw.name.clone());
+                inner_locals.add_name(&kw.name);
             }
             let mut inner_globals = HashSet::default();
             let mut inner_nonlocals = HashSet::default();
@@ -13524,7 +13524,7 @@ fn collect_inner_free_impl(
             // needed-from-outer regardless of whether `outer_bindings`
             // knows about it yet (it'll be promoted on the way down).
             for n in &inner_nonlocals {
-                out.insert(n.clone());
+                out.add_name(n);
             }
             // Reads inside the inner that aren't locals there →
             // candidates for promotion.
@@ -13632,7 +13632,7 @@ fn collect_inner_free_impl(
             // enclosing function's `__firstlineno__` a cell even though
             // the body never reads it).
             for n in &class_nonlocals {
-                from_body.insert(n.clone());
+                from_body.add_name(n);
             }
             // The nested scopes are analyzed as *the class body* sees
             // them, not as this scope does: an inlined comprehension in
@@ -14824,19 +14824,19 @@ fn collect_inner_free_expr(
         ExprKind::Lambda { args, body } | ExprKind::TypeParamFn { args, body } => {
             let mut inner_locals: HashSet<String> = HashSet::default();
             for a in &args.posonlyargs {
-                inner_locals.insert(a.name.clone());
+                inner_locals.add_name(&a.name);
             }
             for a in &args.args {
-                inner_locals.insert(a.name.clone());
+                inner_locals.add_name(&a.name);
             }
             if let Some(va) = &args.vararg {
-                inner_locals.insert(va.name.clone());
+                inner_locals.add_name(&va.name);
             }
             for a in &args.kwonlyargs {
-                inner_locals.insert(a.name.clone());
+                inner_locals.add_name(&a.name);
             }
             if let Some(kw) = &args.kwarg {
-                inner_locals.insert(kw.name.clone());
+                inner_locals.add_name(&kw.name);
             }
             let mut reads = HashSet::default();
             collect_reads_deep(body, &mut reads);
@@ -15044,7 +15044,7 @@ fn collect_self_attr_stores(stmts: &[Stmt], self_name: &str, out: &mut HashSet<S
         match &e.kind {
             ExprKind::Attribute { value, attr } => {
                 if matches!(&value.kind, ExprKind::Name(n) if n == self_name) {
-                    out.insert(attr.clone());
+                    out.add_name(attr);
                 }
             }
             ExprKind::Tuple(elts) | ExprKind::List(elts) => {
@@ -15504,7 +15504,7 @@ fn collect_assigned(stmt: &Stmt, out: &mut HashSet<String>) {
         | StmtKind::AsyncFunctionDef { name, .. }
         | StmtKind::ClassDef { name, .. }
         | StmtKind::TypeAlias { name, .. } => {
-            out.insert(name.clone());
+            out.add_name(name);
         }
         StmtKind::Try {
             body,
@@ -15517,7 +15517,7 @@ fn collect_assigned(stmt: &Stmt, out: &mut HashSet<String>) {
             }
             for h in handlers {
                 if let Some(n) = &h.name {
-                    out.insert(n.clone());
+                    out.add_name(n);
                 }
                 for s in &h.body {
                     collect_assigned(s, out);
@@ -15830,7 +15830,7 @@ fn collect_walrus_expr(expr: &Expr, out: &mut HashSet<String>) {
         | ExprKind::SetComp { elt, generators }
         | ExprKind::GeneratorExp { elt, generators } => {
             collect_comp_scope_walruses(elt, None, generators, &mut |n| {
-                out.insert(n.to_owned());
+                out.add_name(n);
             });
         }
         ExprKind::DictComp {
@@ -15839,7 +15839,7 @@ fn collect_walrus_expr(expr: &Expr, out: &mut HashSet<String>) {
             generators,
         } => {
             collect_comp_scope_walruses(key, Some(value), generators, &mut |n| {
-                out.insert(n.to_owned());
+                out.add_name(n);
             });
         }
         ExprKind::Yield(value) => {
@@ -15875,7 +15875,7 @@ fn collect_global_decls_deep(stmt: &Stmt, out: &mut HashSet<String>) {
     match &stmt.kind {
         StmtKind::Global(ns) => {
             for n in ns {
-                out.insert(n.clone());
+                out.add_name(n);
             }
         }
         StmtKind::FunctionDef { body, .. }
@@ -16279,7 +16279,7 @@ fn function_free_names(args: &AstArguments, body: &[Stmt]) -> HashSet<String> {
         .chain(&args.vararg)
         .chain(&args.kwarg)
     {
-        nested_locals.insert(a.name.clone());
+        nested_locals.add_name(&a.name);
     }
     let mut nested_globals = HashSet::default();
     let mut nested_nonlocals = HashSet::default();
@@ -16429,7 +16429,7 @@ fn class_body_reads_dunder(body: &[Stmt], name: &str) -> bool {
             // function.
             _ => {
                 if stmt_reads_dunder_at_class_level(stmt, name) {
-                    out.insert(name.to_owned());
+                    out.add_name(name);
                 }
             }
         }
@@ -16512,12 +16512,12 @@ fn collect_decls(
     match &stmt.kind {
         StmtKind::Global(ns) => {
             for n in ns {
-                globals.insert(n.clone());
+                globals.add_name(n);
             }
         }
         StmtKind::Nonlocal(ns) => {
             for n in ns {
-                nonlocals.insert(n.clone());
+                nonlocals.add_name(n);
             }
         }
         StmtKind::Assign { targets, .. } => {
@@ -16583,7 +16583,7 @@ fn collect_decls(
         | StmtKind::AsyncFunctionDef { name, .. }
         | StmtKind::ClassDef { name, .. }
         | StmtKind::TypeAlias { name, .. } => {
-            assigned.insert(name.clone());
+            assigned.add_name(name);
         }
         StmtKind::Try {
             body,
@@ -16596,7 +16596,7 @@ fn collect_decls(
             }
             for h in handlers {
                 if let Some(n) = &h.name {
-                    assigned.insert(n.clone());
+                    assigned.add_name(n);
                 }
                 for s in &h.body {
                     collect_decls(s, globals, nonlocals, assigned);
@@ -16731,7 +16731,7 @@ fn collect_target_names_ordered(expr: &Expr, out: &mut Vec<String>) {
 fn collect_target_names(expr: &Expr, out: &mut HashSet<String>) {
     match &expr.kind {
         ExprKind::Name(n) => {
-            out.insert(n.clone());
+            out.add_name(n);
         }
         ExprKind::Tuple(items) | ExprKind::List(items) => {
             for x in items {
@@ -16757,7 +16757,7 @@ fn collect_pattern_names(pat: &Pattern, out: &mut HashSet<String>) {
         | PatternKind::Capture(None)
         | PatternKind::Star(None) => {}
         PatternKind::Capture(Some(n)) | PatternKind::Star(Some(n)) => {
-            out.insert(n.clone());
+            out.add_name(n);
         }
         PatternKind::Sequence(items) => {
             for p in items {
@@ -16769,7 +16769,7 @@ fn collect_pattern_names(pat: &Pattern, out: &mut HashSet<String>) {
                 collect_pattern_names(p, out);
             }
             if let Some(Some(n)) = rest {
-                out.insert(n.clone());
+                out.add_name(n);
             }
         }
         PatternKind::Class {
@@ -16790,7 +16790,7 @@ fn collect_pattern_names(pat: &Pattern, out: &mut HashSet<String>) {
             }
         }
         PatternKind::As { pattern, name } => {
-            out.insert(name.clone());
+            out.add_name(name);
             collect_pattern_names(pattern, out);
         }
     }
@@ -17488,7 +17488,7 @@ fn collect_comp_reads(
 fn collect_reads_expr(expr: &Expr, out: &mut HashSet<String>) {
     match &expr.kind {
         ExprKind::Name(n) => {
-            out.insert(n.clone());
+            out.add_name(n);
         }
         ExprKind::Attribute { value, .. } | ExprKind::Starred(value) => {
             collect_reads_expr(value, out);
diff --git a/crates/weavepy-compiler/src/validate.rs b/crates/weavepy-compiler/src/validate.rs
index d6a50a6c..e5ee0b3d 100644
--- a/crates/weavepy-compiler/src/validate.rs
+++ b/crates/weavepy-compiler/src/validate.rs
@@ -16,6 +16,7 @@ use weavepy_parser::ast::{
     PatternKind, Stmt, StmtKind,
 };
 
+use crate::fxhash::NameSet;
 use crate::CompileError;
 
 /// `__future__` features understood by CPython 3.13. All are mandatory
@@ -168,7 +169,7 @@ impl Validator<'_> {
 
     /// Record a *read* of `name` in the current scope (CPython `USE`).
     fn mark_use(&mut self, name: &str) {
-        self.scope_mut().used.insert(name.to_owned());
+        self.scope_mut().used.add_name(name);
     }
 
     /// Record a binding of `name` (CPython `DEF_LOCAL`). Comprehension
@@ -180,7 +181,7 @@ impl Validator<'_> {
             .iter()
             .rposition(|s| s.kind != ScopeKind::Comprehension)
             .expect("scope stack always has a non-comprehension scope");
-        self.scopes[idx].assigned.insert(name.to_owned());
+        self.scopes[idx].assigned.add_name(name);
     }
 
     /// Visit an annotation expression. Under PEP 563 the annotation is
@@ -583,7 +584,7 @@ impl Validator<'_> {
                                 }
                             }
                             let n = n.clone();
-                            self.scope_mut().annotated.insert(n.clone());
+                            self.scope_mut().annotated.add_name(&n);
                             self.mark_assigned(&n);
                         } else {
                             let n = n.clone();

From b168c2e9436f2889d5b16afb28edc6768fa8ac1b Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 00:37:07 -0700
Subject: [PATCH 152/173] perf: specialize keyword call sites from the core
 loop

The core loop takes every execution of a CALL_KW site, but only the full
step handler ever recorded the site's CallPyKwNames permutation, so the
in-place pure-leaf and inline-activation keyword calls never applied in
interpreted code: each keyword call bound its arguments generically and
ran the callee in a nested interpreter loop. The core loop's call lane
now makes the same decision (decide when empty, count a cooldown down,
cool a site that stops binding).

A keyword call of a plain class now constructs in place too
(core_new_kw): the arguments bind by name as the generic binder would,
and the __init__ activation runs inline, as a positional construction's
does. Anything unusual (a custom __new__ or metaclass __call__, replaced
defaults, a binder error) still takes the generic path.

A latent bug this made reachable is fixed: the core loop's CALL_KW arm
released its operands without checking for a queued finalizer, so a
keyword argument's __del__ ran late. (The slot-15 permutation bug this
also exposes is fixed by the earlier "keep keyword permutations off
slot 15".)

Measured (instructions retired, macOS, interpreter only):
- f(1, b=2, c=3): 4785 -> 1245; o.m(1, b=2): 4781 -> 933
- C(1, b=2) with a Python __init__: 7556 -> 3239
- super().__init__(a, b=1) in an __init__: 8384 -> 5668
- class_creation probe per iteration: 4.85M -> 4.72M
---
 crates/weavepy-vm/src/lib.rs                  | 226 +++++++++++++++++-
 crates/weavepy-vm/src/specialize.rs           |  40 +++-
 .../test_call_kw_core_specialization.py       | 186 ++++++++++++++
 tests/regrtest/test_class_call_keywords.py    | 196 +++++++++++++++
 4 files changed, 638 insertions(+), 10 deletions(-)
 create mode 100644 tests/regrtest/test_call_kw_core_specialization.py
 create mode 100644 tests/regrtest/test_class_call_keywords.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index f8e0952f..3c1c29d6 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -13652,25 +13652,34 @@ impl Interpreter {
                             // `CALL`'s does (see `core_call_kw`); a common
                             // builtin function (`sorted(xs, key=f)`) runs in
                             // the builtin lane.
-                            let builtin = match &ops[0] {
-                                Object::Function(_) => false,
+                            // (0: a Python function, 1: a builtin, 2: a
+                            // plain class, constructed in place.)
+                            let kind = match &ops[0] {
+                                Object::Function(_) => 0,
                                 Object::Builtin(b)
                                     if !b.binds_instance
                                         && matches!(&ops[1], Object::Unbound)
                                         && builtin_fn_lane_ok(b.name) =>
                                 {
-                                    true
+                                    1
                                 }
+                                Object::Type(t) if !t.flags.is_builtin => 2,
                                 _ => break None,
                             };
                             // SAFETY: `len <= cap`, every slot initialized.
                             unsafe { frame.stack.set_len(len) };
                             frame.pc = pc as u32;
                             *last_pc = last;
-                            let ran = if builtin {
-                                self.core_builtin_kw_lane(sw, pc)
-                            } else {
-                                self.core_call_kw(sw, pc) || self.core_call_lane(sw)
+                            let ran = match kind {
+                                1 => self.core_builtin_kw_lane(sw, pc),
+                                2 => {
+                                    let new = self.core_new_kw(sw, pc, snap_gen);
+                                    if new {
+                                        burst_stats::note_call(burst_stats::CALL_NEW);
+                                    }
+                                    new || self.core_call_lane(sw)
+                                }
+                                _ => self.core_call_kw(sw, pc) || self.core_call_lane(sw),
                             };
                             if !ran && sw.pending.is_none() {
                                 sw.pending = Some(CoreExit::Stop(LeafStop::Step));
@@ -13686,6 +13695,8 @@ impl Interpreter {
                         len = start + 1;
                         last = pc;
                         pc += 1;
+                        // (An argument the call held last died.)
+                        after_release!();
                     }
                     OpCode::Call => {
                         let argc = ins.arg as usize;
@@ -16556,6 +16567,15 @@ impl Interpreter {
         if builtin && !kwnames.is_empty() {
             return false;
         }
+        // A keyword call of a plain function decides the site's
+        // `CallPyKwNames` specialization here, as the full handler does:
+        // the core loop takes every execution of the site, so without it
+        // the in-place and inline keyword calls would never apply.
+        if ins.op == OpCode::CallKw {
+            if let Object::Function(f) = &frame.stack[callee_at] {
+                Self::core_specialize_call_kw(&frame.code, pc, f, args.len(), &kwnames);
+            }
+        }
         // Committed: the operands leave the stack.
         let callee = frame.stack[callee_at].clone();
         let operands: Vec<Object> = frame.stack.drain(callee_at..).collect();
@@ -16608,6 +16628,69 @@ impl Interpreter {
         true
     }
 
+    /// The `CALL_KW` handler's specialization step for the site at `pc` of
+    /// `code`, calling the plain function `f` with `eff_argc` positionals
+    /// (a receiver included) and the keyword `names`: an empty site
+    /// decides, a cooling one counts down, and a specialized one that no
+    /// longer binds this call cools down.
+    #[cold]
+    #[inline(never)]
+    fn core_specialize_call_kw(
+        code: &CodeObject,
+        pc: usize,
+        f: &Rc<PyFunction>,
+        eff_argc: usize,
+        names: &[Object],
+    ) {
+        use weavepy_compiler::InlineCache as IC;
+        let kw_op = OpCode::CallKw as u8;
+        if crate::trace::any_observers_active()
+            && crate::trace::monitoring_union_mask_cached()
+                & crate::trace::event_mask(crate::trace::EVENT_CALL)
+                != 0
+        {
+            return;
+        }
+        let at = pc as u32;
+        match code.caches.get(at) {
+            IC::Empty => {
+                specialize::record_specialize_attempt(kw_op);
+                let decision = specialize::attempt_specialize_call_kw_names(f, eff_argc, names);
+                code.caches.set(at, decision);
+                if matches!(decision, IC::Cooldown(_)) {
+                    specialize::record_specialize_skip(kw_op);
+                } else {
+                    specialize::record_specialize_success(kw_op);
+                }
+            }
+            IC::Cooldown(n) => {
+                code.caches.set(
+                    at,
+                    if n > 0 {
+                        IC::Cooldown(n - 1)
+                    } else {
+                        IC::Empty
+                    },
+                );
+            }
+            IC::CallPyKwNames {
+                func_id,
+                perm,
+                argc,
+                kwc,
+            } => {
+                let binds = argc as usize == eff_argc
+                    && kwc as usize == names.len()
+                    && Self::kw_names_bind_check(f, func_id, perm, names, eff_argc).is_some();
+                if !binds {
+                    specialize::record_miss(kw_op);
+                    code.caches.set(at, IC::Cooldown(COOLDOWN));
+                }
+            }
+            _ => {}
+        }
+    }
+
     /// `FOR_ITER` at the running activation's pc over a native adapter
     /// (`map`, `filter`, an `itertools` object) whose step can run Python
     /// code (`map(f, xs)` with a Python `f`): the step runs through the
@@ -17380,6 +17463,135 @@ impl Interpreter {
         false
     }
 
+    /// `cls(..., name=value)` (the `CALL_KW` at `pc` of `sw`'s synced
+    /// running activation) of a plain class whose `__init__` a lean
+    /// activation can run: the arguments bind by name as the generic
+    /// binder would (see [`Self::lean_bind_keywords`]), and the `__init__`
+    /// activation runs inline as [`Self::core_new_n`]'s does. `false`
+    /// touches nothing.
+    #[inline(never)]
+    fn core_new_kw(&mut self, sw: &mut CoreSwitch, pc: usize, snap_gen: u64) -> bool {
+        if !self.inline_calls_ok()
+            || crate::stdlib::tracemalloc_real::is_tracking()
+            || crate::stdlib::testinternalcapi_mod::reftrace_print_active()
+            || crate::stdlib::testinternalcapi_mod::nomem_alloc_fails()
+            || self.lean_snapshot() != Some(snap_gen)
+        {
+            return false;
+        }
+        // SAFETY: see `CoreSwitch`.
+        let frame = unsafe { &mut *sw.cur };
+        let Some(argc) = frame.code.instructions.get(pc).map(|i| i.arg as usize) else {
+            return false;
+        };
+        let len = frame.stack.len();
+        let Some(Object::Tuple(names)) = frame.stack.last() else {
+            return false;
+        };
+        let kwc = names.len();
+        // Stack: class, self-or-null, positionals, keyword values, names.
+        let Some(callee_slot) = len.checked_sub(kwc + argc + 3) else {
+            return false;
+        };
+        let Object::Type(ty) = &frame.stack[callee_slot] else {
+            return false;
+        };
+        if !matches!(frame.stack[callee_slot + 1], Object::Unbound)
+            || ty.flags.is_builtin
+            || ty.native_kind.get() != 0
+            || !Self::metaclass_calls_like_type(ty)
+        {
+            return false;
+        }
+        let ty = ty.clone();
+        let plan = self.instance_plan(&ty);
+        if plan.seeds_exception_args {
+            return false;
+        }
+        let Some((init, code)) = plan.lean_init.as_ref() else {
+            return false;
+        };
+        // SAFETY: GIL-serialized raw read of the function's code cell;
+        // only the pointer is compared.
+        if !std::ptr::eq(
+            unsafe { Rc::as_ptr(&*init.code.as_ptr()) },
+            Rc::as_ptr(code),
+        ) || !Self::lean_code_ok(code)
+            // The instance binds to the first positional parameter.
+            || code.arg_count == 0
+            || (init.defaults_maybe_overridden()
+                && (init.slot("__defaults__").is_some() || init.slot("__kwdefaults__").is_some()))
+        {
+            return false;
+        }
+        let Some(cells) = init.lean_cells_ref(code) else {
+            return false;
+        };
+        let cells: *const Rc<Vec<Rc<RefCell<Object>>>> = cells;
+        let act = self.inline_slot();
+        // SAFETY: a parked slot's locals storage is its own, and empty.
+        let locals = unsafe { &mut *act.frame.locals.as_ptr() };
+        // (The instance isn't made until the binding holds: `None` keeps
+        // its parameter's place.)
+        let receiver = Object::None;
+        let values = &frame.stack[len - 1 - kwc..len - 1];
+        let bound = Self::lean_bind_keywords(
+            init,
+            code,
+            std::iter::once(&receiver).chain(&frame.stack[callee_slot + 2..len - 1 - kwc]),
+            names.iter().zip(values),
+            locals,
+        );
+        let guard = match (bound, crate::recursion::enter_with(sw.depth_cell)) {
+            (Some(()), crate::recursion::Enter::Ok(guard)) => guard,
+            _ => {
+                locals.clear();
+                self.inline_unslot(act);
+                return false;
+            }
+        };
+        // Committed.
+        let (init, code) = (init.clone(), code.clone());
+        let (inst, tracked) = self.alloc_lean_instance_obj(&ty, &plan.native);
+        drop(plan);
+        locals[0] = inst.clone();
+        if tracked && gc_trace::maybe_auto_collect() {
+            self.run_pending_finalizers();
+        }
+        // SAFETY: as above (finalizers ran against the synced frame).
+        let frame = unsafe { &mut *sw.cur };
+        // The operands leave the stack, as the full call handler releases
+        // them once the parameters are bound.
+        self.release_call_operands(&mut frame.stack, callee_slot);
+        drop(ty);
+        fill_unbound(locals, code.varnames.len());
+        frame.pc = pc as u32 + 1;
+        let mut tmp = None;
+        // SAFETY: see `CoreSwitch`.
+        let mut act = unsafe {
+            let depth = (*sw.inl).len();
+            let (frame, _, shell) = sw.activation(depth, &mut tmp);
+            self.inline_bind_cells(
+                &mut *frame,
+                &mut *shell.cast::<QuietShell<'_>>(),
+                pc,
+                act,
+                code,
+                Object::Function(init),
+                guard,
+                cells,
+            )
+        };
+        act.init_inst = Some(inst);
+        let callee: *mut Frame = &raw mut *act.frame;
+        // SAFETY: as above.
+        unsafe { push_fast(&mut *sw.inl, act) };
+        sw.cur = callee;
+        sw.scratch = usize::MAX;
+        sw.last = &raw mut sw.scratch;
+        true
+    }
+
     /// [`Self::core_new`] of the class `argc` positional arguments below
     /// the top of the stack, called by the instruction at `pc`.
     fn core_new_n(&mut self, sw: &mut CoreSwitch, pc: usize, argc: usize, snap_gen: u64) -> bool {
diff --git a/crates/weavepy-vm/src/specialize.rs b/crates/weavepy-vm/src/specialize.rs
index 5fbebacf..1df0a611 100644
--- a/crates/weavepy-vm/src/specialize.rs
+++ b/crates/weavepy-vm/src/specialize.rs
@@ -522,6 +522,37 @@ pub fn attempt_specialize_call_kw(
     f: &Rc<crate::object::PyFunction>,
     argc: usize,
     kw_names: &[(String, crate::object::Object)],
+) -> InlineCache {
+    specialize_call_kw_by(f, argc, kw_names.iter().map(|(n, _)| n.as_str()))
+}
+
+/// [`attempt_specialize_call_kw`] for keyword names that are the call
+/// site's own `str` objects (the core loop's `CALL_KW` lane).
+pub fn attempt_specialize_call_kw_names(
+    f: &Rc<crate::object::PyFunction>,
+    argc: usize,
+    names: &[crate::object::Object],
+) -> InlineCache {
+    if !names
+        .iter()
+        .all(|n| matches!(n, crate::object::Object::Str(_)))
+    {
+        return InlineCache::Cooldown(COOLDOWN);
+    }
+    specialize_call_kw_by(
+        f,
+        argc,
+        names.iter().map(|n| match n {
+            crate::object::Object::Str(s) => &**s,
+            _ => "",
+        }),
+    )
+}
+
+fn specialize_call_kw_by<'a>(
+    f: &Rc<crate::object::PyFunction>,
+    argc: usize,
+    names: impl ExactSizeIterator<Item = &'a str>,
 ) -> InlineCache {
     let code = f.code();
     if code.is_generator
@@ -531,12 +562,13 @@ pub fn attempt_specialize_call_kw(
     {
         return InlineCache::Cooldown(COOLDOWN);
     }
-    match kw_perm(f, &code, argc, kw_names.iter().map(|(n, _)| n.as_str())) {
+    let kwc = names.len();
+    match kw_perm(f, &code, argc, names) {
         Some((perm, _)) => InlineCache::CallPyKwNames {
             func_id: rc_id(f),
             perm,
             argc: argc as u8,
-            kwc: kw_names.len() as u8,
+            kwc: kwc as u8,
         },
         None => InlineCache::Cooldown(COOLDOWN),
     }
@@ -596,7 +628,9 @@ pub fn kw_perm<'a>(
             }
             None => return None,
         };
-        if covered & (1 << slot) != 0 {
+        // Nibble 15 means "collected by `**kwargs`" (see `KW_TO_VARKW`):
+        // a keyword can't name the sixteenth parameter slot.
+        if covered & (1 << slot) != 0 || slot as u32 == KW_TO_VARKW {
             return None;
         }
         perm |= (slot as u32) << (4 * j);
diff --git a/tests/regrtest/test_call_kw_core_specialization.py b/tests/regrtest/test_call_kw_core_specialization.py
new file mode 100644
index 00000000..0e457529
--- /dev/null
+++ b/tests/regrtest/test_call_kw_core_specialization.py
@@ -0,0 +1,186 @@
+"""Keyword call sites specialized from the core loop keep exact binding
+semantics: a site that changes callee, defaults replaced after the site
+specialized, binder errors, keyword-only and `**kwargs` callees, and
+argument lifetimes."""
+
+import gc
+
+
+def run(fn, n=300):
+    for _ in range(n):
+        fn()
+
+
+# One site, a callee that changes shape between executions.
+def f_ab(a, b=2, c=3):
+    return ("ab", a, b, c)
+
+
+def f_ba(a, c=30, b=20):
+    return ("ba", a, b, c)
+
+
+def f_kwargs(a, **kw):
+    return ("kw", a, sorted(kw.items()))
+
+
+def f_kwonly(a, *, b, c=7):
+    return ("ko", a, b, c)
+
+
+def f_args(a, *rest, b=0, c=0):
+    return ("args", a, rest, b, c)
+
+
+def call_site(f):
+    return f(1, b=2, c=3)
+
+
+expected = {
+    f_ab: ("ab", 1, 2, 3),
+    f_ba: ("ba", 1, 2, 3),
+    f_kwargs: ("kw", 1, [("b", 2), ("c", 3)]),
+    f_kwonly: ("ko", 1, 2, 3),
+    f_args: ("args", 1, (), 2, 3),
+}
+order = [f_ab] * 50 + [f_ba] * 50 + [f_kwargs] * 50 + [f_kwonly] * 50 + [f_args] * 50
+for _ in range(4):
+    for f in order:
+        assert call_site(f) == expected[f], (f, call_site(f))
+    order.reverse()
+
+
+# Defaults replaced after the site has run many times.
+def g(a, b=2, c=3):
+    return (a, b, c)
+
+
+def call_g():
+    return g(1, b=5)
+
+
+for _ in range(300):
+    assert call_g() == (1, 5, 3)
+g.__defaults__ = (20, 30)
+for _ in range(50):
+    assert call_g() == (1, 5, 30)
+g.__defaults__ = None
+for _ in range(50):
+    try:
+        call_g()
+    except TypeError as e:
+        assert "missing 1 required positional argument: 'c'" in str(e), str(e)
+    else:
+        raise AssertionError("expected TypeError")
+g.__defaults__ = (2, 3)
+g.__kwdefaults__ = None
+for _ in range(50):
+    assert call_g() == (1, 5, 3)
+
+
+# Binder errors keep CPython's messages at a warm site.
+def h(a, b=2):
+    return (a, b)
+
+
+def bad_dup():
+    return h(1, a=2)
+
+
+def bad_name():
+    return h(1, z=2)
+
+
+for _ in range(200):
+    assert h(1, b=3) == (1, 3)
+for fn, msg in [
+    (bad_dup, "h() got multiple values for argument 'a'"),
+    (bad_name, "h() got an unexpected keyword argument 'z'"),
+]:
+    for _ in range(100):
+        try:
+            fn()
+        except TypeError as e:
+            assert str(e) == msg, str(e)
+        else:
+            raise AssertionError("expected TypeError")
+
+
+# Positional-only parameters can't bind by keyword; `**kwargs` collects
+# the name instead.
+def po(a, /, **kw):
+    return (a, kw)
+
+
+def call_po():
+    return po(1, a=2)
+
+
+for _ in range(300):
+    assert call_po() == (1, {"a": 2})
+
+
+# A keyword naming the sixteenth parameter slot (as
+# `dataclasses.make_dataclass(..., decorator=...)` does).
+def sixteen(p0, p1, *, k2=2, k3=3, k4=4, k5=5, k6=6, k7=7, k8=8, k9=9,
+            k10=10, k11=11, k12=12, k13=13, k14=14, last=None):
+    return (p0, p1, k14, last)
+
+
+def call_sixteen():
+    return sixteen(0, 1, k14="a", last="b")
+
+
+for _ in range(300):
+    assert call_sixteen() == (0, 1, "a", "b")
+
+
+# Methods called with keywords through the elided self slot.
+class K:
+    def m(self, a, b=0, c=0):
+        return (self.tag, a, b, c)
+
+    def __init__(self, tag):
+        self.tag = tag
+
+
+ks = [K(i) for i in range(3)]
+
+
+def call_m(k):
+    return k.m(1, c=3)
+
+
+for _ in range(200):
+    for k in ks:
+        assert call_m(k) == (k.tag, 1, 0, 3)
+
+
+# A keyword argument's last reference dies with the call.
+class Tracked:
+    alive = 0
+
+    def __init__(self):
+        Tracked.alive += 1
+
+    def __del__(self):
+        Tracked.alive -= 1
+
+
+def take(a, b=None):
+    return a
+
+
+def call_take():
+    take(1, b=Tracked())
+    return Tracked.alive
+
+
+gc.disable()
+try:
+    for _ in range(300):
+        assert call_take() == 0
+finally:
+    gc.enable()
+
+print("ok")
diff --git a/tests/regrtest/test_class_call_keywords.py b/tests/regrtest/test_class_call_keywords.py
new file mode 100644
index 00000000..7067e6ec
--- /dev/null
+++ b/tests/regrtest/test_class_call_keywords.py
@@ -0,0 +1,196 @@
+"""Constructing classes with keyword arguments keeps the generic binder's
+semantics: defaults, keyword-only and variadic `__init__`s, binder error
+messages, `__init__` returning non-None, a changed `__init__` or
+`__defaults__`, metaclass `__call__`, abstract classes, custom `__new__`,
+and argument lifetimes."""
+
+import abc
+import gc
+
+
+class Point:
+    def __init__(self, x, y=0, z=0):
+        self.x = x
+        self.y = y
+        self.z = z
+
+
+def make_point():
+    return Point(1, z=3)
+
+
+for _ in range(300):
+    p = make_point()
+    assert (p.x, p.y, p.z) == (1, 0, 3)
+
+
+class KwOnly:
+    def __init__(self, a, *, b, c=5):
+        self.v = (a, b, c)
+
+
+class Var:
+    def __init__(self, *args, **kw):
+        self.v = (args, sorted(kw.items()))
+
+
+class VarSelf:
+    def __init__(self, a, *rest, **kw):
+        self.v = (a, rest, sorted(kw.items()))
+
+
+for _ in range(300):
+    assert KwOnly(1, b=2).v == (1, 2, 5)
+    assert KwOnly(a=1, b=2, c=3).v == (1, 2, 3)
+    assert Var(1, b=2).v[1] == [("b", 2)]
+    assert len(Var(1, b=2).v[0]) == 1
+    assert VarSelf(1, 2, k=3).v == (1, (2,), [("k", 3)])
+
+
+def expect_type_error(fn, msg):
+    for _ in range(100):
+        try:
+            fn()
+        except TypeError as e:
+            assert str(e) == msg, (str(e), msg)
+        else:
+            raise AssertionError("expected TypeError: " + msg)
+
+
+expect_type_error(lambda: Point(1, w=2), "Point.__init__() got an unexpected keyword argument 'w'")
+expect_type_error(lambda: Point(1, x=2), "Point.__init__() got multiple values for argument 'x'")
+expect_type_error(lambda: Point(self=1, x=2), "Point.__init__() got multiple values for argument 'self'")
+expect_type_error(lambda: Point(y=2), "Point.__init__() missing 1 required positional argument: 'x'")
+expect_type_error(lambda: KwOnly(1, c=2), "KwOnly.__init__() missing 1 required keyword-only argument: 'b'")
+
+
+class ReturnsValue:
+    def __init__(self, a=0):
+        self.a = a
+        return a
+
+
+expect_type_error(lambda: ReturnsValue(a=1), "__init__() should return None, not 'int'")
+for _ in range(50):
+    assert ReturnsValue(a=None).a is None
+
+
+# `__init__` and its defaults changed after the site warmed up.
+class Changing:
+    def __init__(self, a, b=1):
+        self.v = ("first", a, b)
+
+
+def make_changing():
+    return Changing(1, b=2)
+
+
+for _ in range(300):
+    assert make_changing().v == ("first", 1, 2)
+
+
+def second(self, a, b=10, c=20):
+    self.v = ("second", a, b, c)
+
+
+Changing.__init__ = second
+for _ in range(100):
+    assert make_changing().v == ("second", 1, 2, 20)
+second.__defaults__ = (100, 200)
+for _ in range(100):
+    assert make_changing().v == ("second", 1, 2, 200)
+second.__defaults__ = None
+expect_type_error(make_changing, "second() missing 1 required positional argument: 'c'")
+
+
+# A metaclass `__call__` intercepts construction.
+class Meta(type):
+    def __call__(cls, *args, **kw):
+        obj = super().__call__(*args, **kw)
+        obj.via_meta = True
+        return obj
+
+
+class WithMeta(metaclass=Meta):
+    def __init__(self, a, b=0):
+        self.v = (a, b)
+
+
+for _ in range(200):
+    w = WithMeta(1, b=2)
+    assert w.v == (1, 2) and w.via_meta
+
+
+# Abstract classes refuse instantiation whatever the arguments.
+class Base(abc.ABC):
+    @abc.abstractmethod
+    def f(self):
+        pass
+
+    def __init__(self, a=0):
+        self.a = a
+
+
+expect_type_error(
+    lambda: Base(a=1),
+    "Can't instantiate abstract class Base without an implementation for abstract method 'f'",
+)
+
+
+# A custom `__new__` sees the keywords too.
+class WithNew:
+    def __new__(cls, *args, **kw):
+        obj = super().__new__(cls)
+        obj.seen = sorted(kw)
+        return obj
+
+    def __init__(self, a, b=0):
+        self.v = (a, b)
+
+
+for _ in range(200):
+    w = WithNew(1, b=2)
+    assert w.v == (1, 2) and w.seen == ["b"]
+
+
+# Subclass `__init__` chaining with keywords.
+class Child(Point):
+    def __init__(self, x, **kw):
+        super().__init__(x, **kw)
+        self.child = True
+
+
+for _ in range(200):
+    c = Child(1, y=2)
+    assert (c.x, c.y, c.z, c.child) == (1, 2, 0, True)
+
+
+# A keyword argument's last reference dies with the construction.
+class Tracked:
+    alive = 0
+
+    def __init__(self):
+        Tracked.alive += 1
+
+    def __del__(self):
+        Tracked.alive -= 1
+
+
+class Holder:
+    def __init__(self, a, b=None):
+        self.a = a
+
+
+def make_holder():
+    Holder(1, b=Tracked())
+    return Tracked.alive
+
+
+gc.disable()
+try:
+    for _ in range(300):
+        assert make_holder() == 0
+finally:
+    gc.enable()
+
+print("ok")

From f5305dc95893730c8a4512d9b05c467340ccb880 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 01:19:04 -0700
Subject: [PATCH 153/173] perf: run a metaclass's Python __call__ as the call's
 inline method

Calling a class whose metaclass replaces __call__ with a plain Python
function (EnumType does; so does any registry or singleton metaclass)
left the core loop for the slow step, which bound the method and ran it
in a nested interpreter loop. The core loop now turns the call's
operands into that function's method shape, the class in the receiver
slot (as the generic call's bind_method binds it), and takes the inline
call; positional and keyword calls both do. A __call__ that is not a
plain function (staticmethod, classmethod, a callable instance) and a
metaclass that keeps type.__call__ are unaffected.

Measured (instructions retired, macOS, interpreter only):
- a class call through `def __call__(cls, *a, **k): return 1`:
  6970 -> 3709
- Color(1) of an Enum: 13.0k -> 11.6k
---
 crates/weavepy-vm/src/lib.rs                  |  51 +++++
 .../regrtest/test_metaclass_call_dispatch.py  | 175 ++++++++++++++++++
 2 files changed, 226 insertions(+)
 create mode 100644 tests/regrtest/test_metaclass_call_dispatch.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 3c1c29d6..ee909696 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -13672,6 +13672,16 @@ impl Interpreter {
                             *last_pc = last;
                             let ran = match kind {
                                 1 => self.core_builtin_kw_lane(sw, pc),
+                                // A metaclass's own `__call__` takes the
+                                // call as its method (see `core_new`).
+                                // SAFETY: the running activation, synced.
+                                2 if Self::unpack_meta_call(
+                                    unsafe { &mut *sw.cur },
+                                    argc + kwc + 1,
+                                ) =>
+                                {
+                                    self.core_call_kw(sw, pc) || self.core_call_lane(sw)
+                                }
                                 2 => {
                                     let new = self.core_new_kw(sw, pc, snap_gen);
                                     if new {
@@ -17413,9 +17423,50 @@ impl Interpreter {
         else {
             return false;
         };
+        // A class whose metaclass replaces `__call__` with a plain Python
+        // function (`EnumType`): the call is that function's, with the
+        // class as its receiver.
+        // SAFETY: see `CoreSwitch`.
+        if Self::unpack_meta_call(unsafe { &mut *sw.cur }, argc) {
+            return self.core_call(sw, pc);
+        }
         self.core_new_n(sw, pc, argc, snap_gen)
     }
 
+    /// Turn the operands of a call of a class (`argc` positionals, no
+    /// receiver) whose metaclass's `__call__` resolves to a plain Python
+    /// function into that function's call with the class as its receiver,
+    /// as `type(cls).__call__(cls, ...)` binds it (the generic call's
+    /// `bind_method`). `false` touches nothing.
+    fn unpack_meta_call(frame: &mut Frame, argc: usize) -> bool {
+        let n = frame.stack.len();
+        let Some(self_slot) = n.checked_sub(argc + 1) else {
+            return false;
+        };
+        let Some(callee_slot) = self_slot.checked_sub(1) else {
+            return false;
+        };
+        let Object::Type(ty) = &frame.stack[callee_slot] else {
+            return false;
+        };
+        if !matches!(frame.stack[self_slot], Object::Unbound) || Self::metaclass_calls_like_type(ty)
+        {
+            return false;
+        }
+        let Some(meta) = ty.metaclass.borrow().clone() else {
+            return false;
+        };
+        if meta.flags.is_builtin || meta.c_ext_ptr.get() != 0 {
+            return false;
+        }
+        let Some(call @ Object::Function(_)) = meta.lookup("__call__") else {
+            return false;
+        };
+        let cls = std::mem::replace(&mut frame.stack[callee_slot], call);
+        frame.stack[self_slot] = cls;
+        true
+    }
+
     /// `cls(*args)` (the `CALL_FUNCTION_EX` at `pc` of `sw`'s synced
     /// running activation) of a plain class over an exact tuple or list,
     /// with no keywords: the items spread into a call's operands for
diff --git a/tests/regrtest/test_metaclass_call_dispatch.py b/tests/regrtest/test_metaclass_call_dispatch.py
new file mode 100644
index 00000000..566ffc94
--- /dev/null
+++ b/tests/regrtest/test_metaclass_call_dispatch.py
@@ -0,0 +1,175 @@
+"""Calling a class whose metaclass defines `__call__` runs that method
+with the class as its receiver, in positional and keyword calls, through
+inherited, replaced and non-function `__call__`s, and with exceptions
+and argument lifetimes intact."""
+
+import enum
+import gc
+
+
+class Meta(type):
+    def __call__(cls, *args, **kw):
+        return ("meta", cls.__name__, args, sorted(kw.items()))
+
+
+class A(metaclass=Meta):
+    pass
+
+
+class B(A):
+    pass
+
+
+def call_a(c):
+    return c(1, 2)
+
+
+def call_a_kw(c):
+    return c(1, k=2)
+
+
+for _ in range(300):
+    assert call_a(A) == ("meta", "A", (1, 2), [])
+    assert call_a(B) == ("meta", "B", (1, 2), [])
+    assert call_a_kw(A) == ("meta", "A", (1,), [("k", 2)])
+
+
+# Replacing the metaclass's `__call__` after the sites warmed up.
+def new_call(cls, *args, **kw):
+    return ("new", cls.__name__, len(args), len(kw))
+
+
+Meta.__call__ = new_call
+for _ in range(100):
+    assert call_a(A) == ("new", "A", 2, 0)
+    assert call_a_kw(B) == ("new", "B", 1, 1)
+del Meta.__call__
+for _ in range(100):
+    try:
+        call_a(A)
+    except TypeError as e:
+        assert str(e) == "A() takes no arguments", str(e)
+    else:
+        raise AssertionError("expected TypeError")
+
+
+# A `__call__` that is not a plain function is honored as such.
+class StaticMeta(type):
+    __call__ = staticmethod(lambda *args: ("static", args))
+
+
+class S(metaclass=StaticMeta):
+    pass
+
+
+class ClassMeta(type):
+    @classmethod
+    def __call__(mcls, *args):
+        return ("classmethod", mcls.__name__, args)
+
+
+class Cm(metaclass=ClassMeta):
+    pass
+
+
+for _ in range(200):
+    assert S(1) == ("static", (1,))
+    assert Cm(1) == ("classmethod", "ClassMeta", (1,))
+
+
+# A metaclass `__call__` delegating to `type.__call__`.
+class Counting(type):
+    count = 0
+
+    def __call__(cls, *args, **kw):
+        Counting.count += 1
+        return super().__call__(*args, **kw)
+
+
+class P(metaclass=Counting):
+    def __init__(self, x, y=0):
+        self.x, self.y = x, y
+
+
+for i in range(300):
+    p = P(i, y=1) if i % 2 else P(i)
+    assert p.x == i and p.y == i % 2
+assert Counting.count == 300
+
+
+# Exceptions raised inside the metaclass `__call__` propagate with it in
+# the traceback.
+class Raising(type):
+    def __call__(cls, *args):
+        raise ValueError("no " + cls.__name__)
+
+
+class R(metaclass=Raising):
+    pass
+
+
+for _ in range(100):
+    try:
+        R(1)
+    except ValueError as e:
+        assert str(e) == "no R"
+        tb = e.__traceback__
+        names = []
+        while tb is not None:
+            names.append(tb.tb_frame.f_code.co_name)
+            tb = tb.tb_next
+        assert names[-1] == "__call__", names
+    else:
+        raise AssertionError("expected ValueError")
+
+
+# Enum lookups by value go through `EnumType.__call__`.
+class Color(enum.Enum):
+    RED = 1
+    GREEN = 2
+
+
+for _ in range(300):
+    assert Color(1) is Color.RED
+    assert Color(value=2) is Color.GREEN
+try:
+    Color(3)
+except ValueError as e:
+    assert str(e) == "3 is not a valid Color", str(e)
+else:
+    raise AssertionError("expected ValueError")
+
+
+# The argument's last reference dies with the call.
+class Tracked:
+    alive = 0
+
+    def __init__(self):
+        Tracked.alive += 1
+
+    def __del__(self):
+        Tracked.alive -= 1
+
+
+class Drop(type):
+    def __call__(cls, arg):
+        return 1
+
+
+class D(metaclass=Drop):
+    pass
+
+
+def make_d():
+    D(Tracked())
+    return Tracked.alive
+
+
+gc.disable()
+try:
+    for _ in range(300):
+        assert make_d() == 0
+finally:
+    gc.enable()
+
+print("ok")

From c872c6dc2873c0567ca209912ff2d605f34001fd Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:02:32 -0700
Subject: [PATCH 154/173] perf: never specialize an attribute load on a super
 proxy by its class

A `super` proxy resolves attributes along its target's MRO, past the
proxy's class (CPython's super_getattro); WeavePy serves that from
load_attr_instance_default rather than a `__getattribute__` entry, so
type_has_attr_override did not recognize the `super` type, and a
LOAD_ATTR site that had seen a proxy specialized to the `super` type's
own MRO. From its second execution, `super().__new__` on a class
receiver then returned object.__new__ ("object.__new__() takes exactly
one argument"), and any name `object` also defines (__init__, __eq__,
...) could resolve the same wrong way. This only arises where the
compiler emits the generic super() call, as it does for every
zero-argument super() in a module whose top level names `super`.

Super proxy types now count as overriding attribute lookup, and the
core loop's instance-attribute resolver no longer lets `__new__`
through for them.

Correctness fix; no measured performance effect (the specializer and
resolver check a cached type bit only on paths that already decline).
---
 crates/weavepy-vm/src/lib.rs                  |  7 +-
 crates/weavepy-vm/src/specialize.rs           |  4 +-
 .../test_super_proxy_attribute_sites.py       | 70 +++++++++++++++++++
 3 files changed, 78 insertions(+), 3 deletions(-)
 create mode 100644 tests/regrtest/test_super_proxy_attribute_sites.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index ee909696..7aa3b490 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -28719,9 +28719,12 @@ impl Interpreter {
         // `__getattribute__` (a `super` proxy resolves `__init__` in the
         // MRO *after* its own class), where the class-MRO answer the
         // class cache holds would be wrong. `__new__` keeps its historic
-        // exception.
+        // exception, except through a `super` proxy.
         let name = code.names.get(name_idx as usize)?;
-        if name.len() > 4 && name.starts_with("__") && name != "__new__" {
+        if name.len() > 4
+            && name.starts_with("__")
+            && (name != "__new__" || cls.is_super_proxy_type())
+        {
             return None;
         }
         let on_class = match Self::leaf_class_attr(code, cls, name_idx)? {
diff --git a/crates/weavepy-vm/src/specialize.rs b/crates/weavepy-vm/src/specialize.rs
index 1df0a611..1bf371ee 100644
--- a/crates/weavepy-vm/src/specialize.rs
+++ b/crates/weavepy-vm/src/specialize.rs
@@ -717,7 +717,9 @@ pub fn native_fn_id(b: &Rc<crate::object::BuiltinFn>) -> u64 {
 /// would invalidate the simple "dict slot" fast path. We bail out
 /// of LOAD_ATTR / STORE_ATTR specialization for these.
 pub(crate) fn type_has_attr_override(ty: &Rc<TypeObject>) -> bool {
-    if ty.lookup("__getattr__").is_some() {
+    // A `super` proxy resolves attributes along its target's MRO (CPython's
+    // `super_getattro`), never its own class's.
+    if ty.lookup("__getattr__").is_some() || ty.is_super_proxy_type() {
         return true;
     }
     // `object.__getattribute__` lives in `object`'s dict as a sentinel, so a
diff --git a/tests/regrtest/test_super_proxy_attribute_sites.py b/tests/regrtest/test_super_proxy_attribute_sites.py
new file mode 100644
index 00000000..d8cbb4e4
--- /dev/null
+++ b/tests/regrtest/test_super_proxy_attribute_sites.py
@@ -0,0 +1,70 @@
+"""Attribute loads on `super()` proxies resolve along the receiver's MRO
+past the proxy's class at every execution, never along the `super` type's
+own MRO, including names `object` also defines (`__new__`, `__init__`,
+`__eq__`). This module names `super` at top level, so the compiler emits
+the generic `super()` call and an ordinary `LOAD_ATTR` (as CPython does)."""
+
+the_super = super
+
+
+class Base:
+    def __new__(cls, *args):
+        return the_super(Base, cls).__new__(cls)
+
+    def __init__(self, *args):
+        self.args = args
+
+    def __eq__(self, other):
+        return "Base.__eq__"
+
+    def m(self):
+        return "Base.m"
+
+
+class Sub(Base):
+    def __new__(cls, *args):
+        return super().__new__(cls, *args)
+
+    def __init__(self, *args):
+        super().__init__(*args)
+
+    def __eq__(self, other):
+        return super().__eq__(other)
+
+    def m(self):
+        return super().m()
+
+
+class Sub2(Base):
+    def __new__(cls, *args):
+        proxy = super()
+        return proxy.__new__(cls, *args)
+
+    def __init__(self, *args):
+        proxy = super()
+        proxy.__init__(*args)
+
+
+for i in range(300):
+    s = Sub(i)
+    assert type(s) is Sub and s.args == (i,)
+    assert (s == 1) == "Base.__eq__"
+    assert s.m() == "Base.m"
+    s2 = Sub2(i, i)
+    assert type(s2) is Sub2 and s2.args == (i, i)
+
+
+# A `super` subclass as the proxy.
+class MySuper(super):
+    pass
+
+
+class Sub3(Base):
+    def __init__(self, *args):
+        MySuper(Sub3, self).__init__(*args)
+
+
+for i in range(300):
+    assert Sub3(i).args == (i,)
+
+print("ok")

From 6aede41cdcbc21645d104d906452c4a80f574dcd Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:02:40 -0700
Subject: [PATCH 155/173] perf: resolve super() dunders and metaclass receivers
 in the core loop

The core loop's zero-argument super() path (leaf_load_super_attr)
declined two shapes that class machinery uses constantly, sending them
to the slow step and the generic proxy lookup:

- dunders other than __init__ (`super().__setitem__(k, v)` in a dict
  subclass such as enum's EnumDict, `super().__setattr__`, ...), because
  a built-in class may serve a dunder from outside its dict. A dunder now
  resolves when the walk finds it before passing any built-in class (and
  skips introspection-only entries, as the full lookup does);
- a class receiver that is an instance of the super() class, as in a
  metaclass method (`super().__delattr__(name)` in EnumType.__delattr__).
  Its binding is an instance's, through the metaclass's MRO, so it now
  takes the same path. A class receiver that subclasses the super()
  class (class methods, __new__) binds differently and stays on the
  full path, as does `super().__class__`.

Measured (instructions retired, macOS, interpreter only):
- d[k] = v on a dict subclass whose __setitem__ calls super(): 8443 ->
  5958
- super().__len__() in a dict subclass method: 5349 -> 2830
- C.x = 1; del C.x under a metaclass whose __setattr__/__delattr__ call
  super(): 18967 -> 14402
- Enum("E", "RED GREEN BLUE") and use: 1.232M -> 1.185M per iteration
---
 crates/weavepy-vm/src/lib.rs                  |  34 ++-
 .../test_super_dunder_and_metaclass.py        | 229 ++++++++++++++++++
 2 files changed, 259 insertions(+), 4 deletions(-)
 create mode 100644 tests/regrtest/test_super_dunder_and_metaclass.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 7aa3b490..fc142386 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -29190,10 +29190,24 @@ impl Interpreter {
         let Object::Type(cls) = class_obj else {
             return None;
         };
-        let Object::Instance(inst) = self_obj else {
-            return None;
+        // The receiver's class, whose MRO `super()` walks. A class
+        // receiver that is an instance of `cls` (a metaclass method's
+        // `super().__setattr__(...)`) binds like any instance, through
+        // its metaclass; one that is a subclass of `cls` (a class method,
+        // `__new__`) binds unlike one and stays on the full path.
+        let meta_holder;
+        let recv_cls: &TypeObject = match self_obj {
+            Object::Instance(inst) => inst.cls_raw(),
+            Object::Type(k) => {
+                if k.is_subclass_of(cls) {
+                    return None;
+                }
+                // SAFETY: GIL-serialized raw read, as in `plain_metaclass`.
+                meta_holder = unsafe { (*k.metaclass.as_ptr()).clone() }?;
+                &meta_holder
+            }
+            _ => return None,
         };
-        let recv_cls = inst.cls_raw();
         let ver = recv_cls.attr_version.get();
         let cls_id = Rc::as_ptr(cls) as usize as u64;
         let slot = code_method_slot(code, cache_pc);
@@ -29214,9 +29228,14 @@ impl Interpreter {
             _ => {}
         }
         let name = code.names.get(name_idx as usize)?;
-        if name.starts_with("__") && name != "__init__" {
+        // `super().__class__` is the proxy's own attribute.
+        if name == "__class__" {
             return None;
         }
+        // A built-in class can serve a dunder from outside its dict (a
+        // slot wrapper the full lookup synthesizes): a dunder's walk may
+        // not pass one by.
+        let dunder = name.starts_with("__") && name != "__init__";
         // (The receiver's metaclass takes no part: `super()` reads the
         // dicts along its stored MRO, which a custom `mro()` produced.)
         let mro = recv_cls.mro.try_borrow().ok()?;
@@ -29229,9 +29248,16 @@ impl Interpreter {
             }
             let d = owner.dict.try_borrow().ok()?;
             if let Some(v) = d.get(&key) {
+                // (An introspection-only entry is invisible to dispatch.)
+                if owner.flags.is_builtin && crate::descr_registry::is_surface_only(v) {
+                    return None;
+                }
                 found = Some(v.clone());
                 break;
             }
+            if dunder && owner.flags.is_builtin {
+                return None;
+            }
         }
         match found? {
             Object::Function(f) => {
diff --git a/tests/regrtest/test_super_dunder_and_metaclass.py b/tests/regrtest/test_super_dunder_and_metaclass.py
new file mode 100644
index 00000000..47331937
--- /dev/null
+++ b/tests/regrtest/test_super_dunder_and_metaclass.py
@@ -0,0 +1,229 @@
+"""Zero-argument `super()` resolving dunders past built-in bases, and in
+metaclass methods whose receiver is a class: the same attribute CPython
+finds, bound the same way, through class changes after warm-up.
+
+(The module never names `super` at top level, which would make the
+compiler use the generic `super()` call instead of `LOAD_SUPER_ATTR`.)"""
+
+
+class D(dict):
+    def __setitem__(self, k, v):
+        super().__setitem__(k, v * 2)
+
+    def __getitem__(self, k):
+        return super().__getitem__(k) + 1
+
+    def __len__(self):
+        return super().__len__() + 100
+
+    def __eq__(self, other):
+        return super().__eq__(other)
+
+    def __repr__(self):
+        return "D" + super().__repr__()
+
+    def __contains__(self, k):
+        return super().__contains__(k)
+
+
+d = D()
+for i in range(300):
+    d["k"] = i
+    assert d["k"] == 2 * i + 1
+    assert len(d) == 101
+    assert d == {"k": 2 * i}
+    assert repr(d) == "D{'k': %d}" % (2 * i)
+    assert "k" in d and "z" not in d
+
+
+class L(list):
+    def __getitem__(self, i):
+        return super().__getitem__(i) * 10
+
+    def __iter__(self):
+        return super().__iter__()
+
+
+lst = L([1, 2, 3])
+for _ in range(300):
+    assert lst[1] == 20
+    assert list(iter(lst)) == [1, 2, 3]
+
+
+# Dunders that only `object` provides.
+class P:
+    def __init__(self):
+        super().__setattr__("x", 1)
+
+    def __setattr__(self, k, v):
+        super().__setattr__(k, v + 1)
+
+    def __getattribute__(self, k):
+        return super().__getattribute__(k)
+
+    def __eq__(self, other):
+        return super().__eq__(other)
+
+    def __hash__(self):
+        return super().__hash__()
+
+    def __format__(self, spec):
+        return "P" + super().__format__("")[:0]
+
+
+for _ in range(300):
+    p = P()
+    assert p.x == 1
+    p.y = 5
+    assert p.y == 6
+    assert (p == p) is True and (p == 1) is False
+    assert p.__eq__(1) is NotImplemented
+    assert hash(p) == object.__hash__(p)
+    assert format(p, "") == "P"
+
+
+# A user class between the subclass and the built-in base.
+class Mid(dict):
+    def __setitem__(self, k, v):
+        super().__setitem__(k, ("mid", v))
+
+
+class Leaf(Mid):
+    def __setitem__(self, k, v):
+        super().__setitem__(k, ("leaf", v))
+
+
+lf = Leaf()
+for i in range(300):
+    lf["a"] = i
+    assert dict.__getitem__(lf, "a") == ("mid", ("leaf", i))
+
+
+# Metaclass methods: `super()` with a class receiver.
+log = []
+
+
+class Meta(type):
+    def __setattr__(cls, k, v):
+        log.append(("set", k))
+        super().__setattr__(k, v)
+
+    def __delattr__(cls, k):
+        log.append(("del", k))
+        super().__delattr__(k)
+
+    def __getattribute__(cls, k):
+        return super().__getattribute__(k)
+
+    def describe(cls):
+        return super().__repr__()
+
+    def __call__(cls, *args):
+        return super().__call__(*args)
+
+
+class C(metaclass=Meta):
+    def __init__(self, a=0):
+        self.a = a
+
+
+for i in range(300):
+    C.attr = i
+    assert C.attr == i
+    del C.attr
+    assert not hasattr(C, "attr")
+    assert C.describe().startswith("<class ")
+    assert C(i).a == i
+assert log.count(("set", "attr")) == 300 and log.count(("del", "attr")) == 300
+try:
+    del C.missing
+except AttributeError as e:
+    assert "missing" in str(e)
+else:
+    raise AssertionError("expected AttributeError")
+
+
+# A metaclass hierarchy: super() from the derived metaclass reaches the
+# base metaclass's Python method.
+class BaseMeta(type):
+    def tag(cls):
+        return "base:" + cls.__name__
+
+
+class DerivedMeta(BaseMeta):
+    def tag(cls):
+        return "derived>" + super().tag()
+
+
+class T(metaclass=DerivedMeta):
+    pass
+
+
+for _ in range(300):
+    assert T.tag() == "derived>base:T"
+
+
+# Class methods and `__new__`: a class receiver that subclasses the
+# `super()` class.
+class Base:
+    @classmethod
+    def make(cls):
+        return ("base", cls.__name__)
+
+    def __new__(cls, *args):
+        return super().__new__(cls)
+
+
+class Sub(Base):
+    @classmethod
+    def make(cls):
+        return ("sub",) + super().make()
+
+    def __new__(cls, *args):
+        return super().__new__(cls, *args)
+
+
+for _ in range(300):
+    assert Sub.make() == ("sub", "base", "Sub")
+    assert type(Sub(1)) is Sub
+
+
+# `super().__class__` is the proxy's.
+class S:
+    def cls_of_super(self):
+        return super().__class__
+
+
+for _ in range(100):
+    assert S().cls_of_super().__name__ == "super"
+
+
+# The resolution follows a class change made after warm-up.
+class Base2(dict):
+    pass
+
+
+class Child2(Base2):
+    def __setitem__(self, k, v):
+        super().__setitem__(k, v)
+
+
+c2 = Child2()
+for i in range(300):
+    c2["x"] = i
+assert dict.__getitem__(c2, "x") == 299
+
+
+def base2_setitem(self, k, v):
+    dict.__setitem__(self, k, ("patched", v))
+
+
+Base2.__setitem__ = base2_setitem
+for i in range(50):
+    c2["x"] = i
+    assert dict.__getitem__(c2, "x") == ("patched", i)
+del Base2.__setitem__
+c2["x"] = 7
+assert dict.__getitem__(c2, "x") == 7
+
+print("ok")

From 91b90bb643a77e5d8d716fa6ce130f4e79a8d799 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:28:48 -0700
Subject: [PATCH 156/173] perf: remember attribute misses on values of built-in
 types

`hasattr(1, '__get__')` and `getattr(value, name, default)` on a value
of a built-in type ran the full attribute lookup and built, formatted
and discarded an AttributeError every time. enum asks
`hasattr(value, '__get__')` (then `__set__`, `__delete__`) of every
member value while building an Enum, and introspective code probes
builtin values this way constantly.

The values of the built-in types whose attributes all come from their
type (None, bool, int, float, str, bytes, tuple, list, dict, set,
frozenset) resolve an attribute by type and name alone, so a miss
holds for the class's current version. Like the TypeAttrMissing verdict
for built-in type objects, a new ValueAttrMissing verdict in the type
cache records a value's miss: a first miss still takes the full lookup,
a repeat answers at once. Subclass instances, functions, modules and
anything else with attributes of its own are untouched.

Measured (instructions retired, macOS, interpreter only):
- hasattr(1, '__get__'): 8509 -> 2157
- enum._is_descriptor(1): 26.3k -> 8.5k
- Enum("E", "RED GREEN BLUE") and use: 1.185M -> 1.138M per iteration
---
 crates/weavepy-vm/src/lib.rs                  | 49 ++++++++++++++-
 crates/weavepy-vm/src/types.rs                |  5 ++
 .../test_hasattr_builtin_value_misses.py      | 62 +++++++++++++++++++
 3 files changed, 114 insertions(+), 2 deletions(-)
 create mode 100644 tests/regrtest/test_hasattr_builtin_value_misses.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index fc142386..2024974a 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -41442,12 +41442,26 @@ impl Interpreter {
                 ))),
             };
         }
-        if args.len() >= 3 && attr_certainly_missing(&args[0], &name) {
+        if args.len() < 3 {
+            return self.load_attr(&args[0], &name);
+        }
+        if attr_certainly_missing(&args[0], &name) {
             return Ok(args[2].clone());
         }
+        let value_class = builtin_value_class(&args[0]);
+        if let Some(c) = &value_class {
+            if c.verdict_cached(crate::types::Verdict::ValueAttrMissing, &name) == Some(true) {
+                return Ok(args[2].clone());
+            }
+        }
         match self.load_attr(&args[0], &name) {
             Ok(v) => Ok(v),
-            Err(e) if args.len() >= 3 && self.is_attribute_error(&e) => Ok(args[2].clone()),
+            Err(e) if self.is_attribute_error(&e) => {
+                if let Some(c) = value_class {
+                    c.verdict_record(crate::types::Verdict::ValueAttrMissing, &name, true);
+                }
+                Ok(args[2].clone())
+            }
             Err(e) => Err(e),
         }
     }
@@ -41490,12 +41504,21 @@ impl Interpreter {
                 return Ok(Object::Bool(false));
             }
         }
+        let value_class = builtin_value_class(&args[0]);
+        if let Some(c) = &value_class {
+            if c.verdict_cached(crate::types::Verdict::ValueAttrMissing, &name) == Some(true) {
+                return Ok(Object::Bool(false));
+            }
+        }
         match self.load_attr(&args[0], &name) {
             Ok(_) => Ok(Object::Bool(true)),
             Err(e) if self.is_attribute_error(&e) => {
                 if let Some(t) = builtin_type {
                     t.verdict_record(crate::types::Verdict::TypeAttrMissing, &name, true);
                 }
+                if let Some(c) = value_class {
+                    c.verdict_record(crate::types::Verdict::ValueAttrMissing, &name, true);
+                }
                 Ok(Object::Bool(false))
             }
             Err(e) => Err(e),
@@ -69510,6 +69533,28 @@ thread_local! {
     > = std::cell::RefCell::new(crate::fasthash::FxHashMap::default());
 }
 
+/// The built-in class whose attributes are all of `obj`'s: a value of a
+/// built-in type that carries none of its own (no instance dict, no
+/// per-object attribute store), so what `obj.name` resolves to depends on
+/// the class and the name alone. `None` for anything else.
+fn builtin_value_class(obj: &Object) -> Option<Rc<TypeObject>> {
+    matches!(
+        obj,
+        Object::None
+            | Object::Bool(_)
+            | Object::Int(_)
+            | Object::Float(_)
+            | Object::Str(_)
+            | Object::Bytes(_)
+            | Object::Tuple(_)
+            | Object::List(_)
+            | Object::Dict(_)
+            | Object::Set(_)
+            | Object::FrozenSet(_)
+    )
+    .then(|| crate::builtins::class_of(obj))
+}
+
 /// Whether `obj.name` certainly raises `AttributeError` without running
 /// any code: an ordinary instance under the default attribute protocol,
 /// with no class attribute of that name and none in its `__dict__`. A
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index 1875b626..d64c2569 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -171,6 +171,11 @@ pub(crate) enum Verdict {
     /// `hasattr(type(coro), '_asyncio_future_blocking')` for every
     /// coroutine it is handed). See [`TypeObject::verdict_record`].
     TypeAttrMissing = 6,
+    /// A value of the builtin class (one whose attributes all come from
+    /// the class, see `builtin_value_class`) answered `name` with
+    /// `AttributeError`: enum asks `hasattr(value, '__get__')` of every
+    /// member value. Recorded like [`Self::TypeAttrMissing`].
+    ValueAttrMissing = 7,
 }
 
 pub struct AttrVersion(std::sync::atomic::AtomicU64);
diff --git a/tests/regrtest/test_hasattr_builtin_value_misses.py b/tests/regrtest/test_hasattr_builtin_value_misses.py
new file mode 100644
index 00000000..9affa6af
--- /dev/null
+++ b/tests/regrtest/test_hasattr_builtin_value_misses.py
@@ -0,0 +1,62 @@
+"""`hasattr` and `getattr` with a default on values of built-in types
+answer repeated misses the same as the first, without confusing a value
+with its type, a subclass instance, or a hit."""
+
+values = [None, True, 1, 1.5, "s", b"b", (1,), [1], {1: 2}, {1}, frozenset({1})]
+for _ in range(200):
+    for v in values:
+        assert not hasattr(v, "__get__")
+        assert not hasattr(v, "no_such_attribute")
+        assert getattr(v, "__set__", "dflt") == "dflt"
+        assert hasattr(v, "__class__")
+        assert hasattr(v, "__eq__")
+        assert getattr(v, "__class__", None) is type(v)
+
+for _ in range(200):
+    assert hasattr(1, "real") and getattr(1, "imag", None) == 0
+    assert hasattr([], "append") and getattr({}, "get", None) is not None
+    assert hasattr("x", "join") and not hasattr("x", "append")
+    assert hasattr((), "count") and not hasattr((), "append")
+
+# A miss on a value is not a miss on its type, nor on a subclass's
+# instances.
+assert not hasattr(1, "mro")
+assert hasattr(int, "mro")
+
+
+class MyInt(int):
+    pass
+
+
+class Lazy(list):
+    def __getattr__(self, name):
+        return "lazy:" + name
+
+
+m = MyInt(3)
+m.no_such_attribute = "set"
+lz = Lazy()
+for _ in range(200):
+    assert not hasattr(3, "no_such_attribute")
+    assert getattr(m, "no_such_attribute") == "set"
+    assert hasattr(m, "no_such_attribute")
+    assert hasattr(lz, "no_such_attribute")
+    assert getattr(lz, "__get__", None) == "lazy:__get__"
+    assert getattr([], "__get__", None) is None
+
+# Errors other than AttributeError still propagate, and a missing
+# attribute without a default still raises with CPython's message.
+try:
+    getattr(1, "nope")
+except AttributeError as e:
+    assert str(e) == "'int' object has no attribute 'nope'", str(e)
+else:
+    raise AssertionError("expected AttributeError")
+try:
+    hasattr(1, 2)
+except TypeError as e:
+    assert "attribute name must be string" in str(e), str(e)
+else:
+    raise AssertionError("expected TypeError")
+
+print("ok")

From 07179c6394b684467481d89f282f5bb63199dcc5 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:35:39 -0700
Subject: [PATCH 157/173] perf: copy a SharedStr straight into a String in
 to_string

`s.to_string()` on a SharedStr went through the blanket ToString,
formatting the text through Display and Formatter::pad. The VM does
this wherever it turns a str object into an owned name (getattr,
hasattr, setattr and delattr's attribute names, class names, function
attributes). An inherent SharedStr::to_string, which method resolution
prefers, now copies the bytes directly; the result is the same string
(no width or precision is ever set).

Measured (instructions retired, macOS, pinned hash seed): class_creation
probe 3.482M -> 3.466M per iteration.
---
 crates/weavepy-vm/src/shared_value.rs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/crates/weavepy-vm/src/shared_value.rs b/crates/weavepy-vm/src/shared_value.rs
index 189c30f3..560f73b2 100644
--- a/crates/weavepy-vm/src/shared_value.rs
+++ b/crates/weavepy-vm/src/shared_value.rs
@@ -850,6 +850,18 @@ impl SharedStr {
     }
 }
 
+impl SharedStr {
+    /// The text as an owned `String`, copied directly. The blanket
+    /// `ToString` would run it through `Display` and `Formatter::pad`,
+    /// several times the work for the same result (no width is set);
+    /// as an inherent method this one is what `s.to_string()` calls.
+    #[inline]
+    #[allow(clippy::inherent_to_string_shadow_display)]
+    pub fn to_string(&self) -> String {
+        String::from(&**self)
+    }
+}
+
 impl SharedStr {
     /// The shared one-character string of ASCII byte `c` (below `0x80`).
     #[inline]

From 5f8438c6684db039c9c884885e0bde777c7271fd Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:59:49 -0700
Subject: [PATCH 158/173] perf: serve a class __dict__ view's methods through
 the builtin method memo

`cls.__dict__.get(name)` (enum reads its classes this way, as does
inspect) loaded `get` off the mappingproxy on the slow attribute path
and called it through the full handler: a mappingproxy was not among
the builtin receiver kinds method_memo_tag names, which the method
memo, the LOAD_ATTR method slots and the core loop's builtin lane all
key on. A proxy over a class dict now has its own tag; its methods are
plain native bodies over the wrapped dict. A proxy over another
mapping (MappingProxyObj) still calls that mapping's own methods.

Measured (instructions retired, macOS, interpreter only):
- C.__dict__.get('zz'): 3745 -> 1713
- Enum("E", "RED GREEN BLUE") and use: 1.133M -> 1.105M per iteration
---
 crates/weavepy-vm/src/builtins.rs             |  4 ++
 .../regrtest/test_class_dict_proxy_methods.py | 57 +++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 tests/regrtest/test_class_dict_proxy_methods.py

diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs
index 00a6a150..913a64ac 100644
--- a/crates/weavepy-vm/src/builtins.rs
+++ b/crates/weavepy-vm/src/builtins.rs
@@ -517,6 +517,10 @@ pub(crate) fn method_memo_tag(obj: &Object) -> Option<u8> {
         Object::File(f) if f.extra_attrs.borrow().is_empty() => {
             16 + (f.io_kind.get() as u8) * 4 + u8::from(f.binary) * 2 + u8::from(f.is_memory())
         }
+        // A class's `__dict__` view (`cls.__dict__.get(name)`, as enum and
+        // `inspect` read it). A view over another mapping
+        // (`MappingProxyObj`) calls that mapping's own methods instead.
+        Object::MappingProxy(_) => 120,
         _ => return None,
     };
     Some(tag)
diff --git a/tests/regrtest/test_class_dict_proxy_methods.py b/tests/regrtest/test_class_dict_proxy_methods.py
new file mode 100644
index 00000000..b8dacda9
--- /dev/null
+++ b/tests/regrtest/test_class_dict_proxy_methods.py
@@ -0,0 +1,57 @@
+"""Methods of a class's `__dict__` view (a mappingproxy) at warm call
+sites: they read the class's current namespace, refuse writes, and stay
+distinct from a proxy over another mapping."""
+
+import types
+
+
+class A:
+    x = 1
+
+    def f(self):
+        return 2
+
+
+for i in range(300):
+    d = A.__dict__
+    assert d.get("x") == 1
+    assert d.get("missing") is None
+    assert d.get("missing", i) == i
+    assert "f" in d.keys()
+    assert d["x"] == 1
+    assert ("x", 1) in d.items()
+    assert 1 in d.values()
+    assert d.copy()["x"] == 1 and type(d.copy()) is dict
+    assert "x" in d and "nope" not in d
+    assert len(d) == len(dict(d))
+    assert set(iter(d)) == set(d.keys())
+
+# The view follows the class.
+A.y = 5
+for _ in range(100):
+    assert A.__dict__.get("y") == 5
+del A.y
+for _ in range(100):
+    assert A.__dict__.get("y", "gone") == "gone"
+
+# Writes are refused.
+try:
+    A.__dict__["z"] = 1
+except TypeError:
+    pass
+else:
+    raise AssertionError("expected TypeError")
+
+
+# A proxy over a dict subclass calls the subclass's own methods.
+class Loud(dict):
+    def get(self, k, default=None):
+        return ("loud", k)
+
+
+proxy = types.MappingProxyType(Loud(a=1))
+for _ in range(300):
+    assert proxy.get("a") == ("loud", "a")
+    assert A.__dict__.get("x") == 1
+
+print("ok")

From a3fd2c25e21b0c047f4b1a097973e6155a67eead Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 03:12:00 -0700
Subject: [PATCH 159/173] perf: run a Python __setitem__ as an inline
 activation from STORE_SUBSCR

`obj[key] = value` on an instance whose class defines __setitem__ in
Python (a dict subclass such as enum's EnumDict, which every class body
assignment and Enum member goes through, or any custom container)
left the core loop for the slow step, which called the method in a
nested interpreter loop. As BINARY_SUBSCR already does for a Python
__getitem__ (try_inline_getitem), the core loop now runs it as an
inline activation: the three operands become its (self, key, value)
arguments and its result is discarded, as a property setter's is. The
site remembers the function by the class's attribute version; a
__setitem__ with other parameters, a descriptor or a native one keeps
its existing path.

Measured (instructions retired, macOS, interpreter only):
- d[k] = v on a dict subclass whose __setitem__ calls
  super().__setitem__: 5958 -> 3231
- Enum("E", "RED GREEN BLUE") and use: 1.105M -> 1.087M per iteration
---
 crates/weavepy-vm/src/lib.rs                 |  98 +++++++++++++
 tests/regrtest/test_setitem_python_dunder.py | 136 +++++++++++++++++++
 2 files changed, 234 insertions(+)
 create mode 100644 tests/regrtest/test_setitem_python_dunder.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 2024974a..023ffd24 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -10255,6 +10255,75 @@ impl Interpreter {
         Some(self.inline_bind(frame, shell, pc, act, code, Object::Function(f), guard))
     }
 
+    /// `container[key] = value` (the `STORE_SUBSCR` at `pc` of `frame`)
+    /// for an instance whose class's `__setitem__` is a plain Python
+    /// function of `(self, key, value)`, as an inline activation of it
+    /// (CPython calls it from `STORE_SUBSCR` alike): the three operands
+    /// leave the stack as its arguments, and its result is discarded.
+    fn try_inline_setitem(
+        &mut self,
+        frame: &mut Frame,
+        shell: &mut QuietShell<'_>,
+        pc: usize,
+    ) -> Option<Box<InlineAct>> {
+        // Stack (top-down): key, container, value.
+        let n = frame.stack.len();
+        if n < 3 {
+            return None;
+        }
+        let Object::Instance(inst) = &frame.stack[n - 2] else {
+            return None;
+        };
+        // The site remembers the function by the class's attribute version.
+        let cls = inst.cls_raw();
+        let ver = cls.attr_version.get();
+        let slot = code_method_slot(&frame.code, pc as u32);
+        let f = match slot.and_then(|s| s.get_held(ver)) {
+            Some(f) => f,
+            None => {
+                let Some(Object::Function(f)) = cls.lookup("__setitem__") else {
+                    return None;
+                };
+                if let Some(s) = slot {
+                    s.set(ver, &f);
+                }
+                f
+            }
+        };
+        let code = f.code();
+        if code.arg_count != 3
+            || code.kwonly_count != 0
+            || code.has_varargs
+            || code.has_varkeywords
+            || code.is_generator
+            || code.is_coroutine
+            || code.is_async_generator
+            || !Self::lean_code_ok(&code)
+        {
+            return None;
+        }
+        f.lean_cells_ref(&code)?;
+        // Past the recursion limit the nested lean path raises.
+        let crate::recursion::Enter::Ok(guard) = crate::recursion::enter() else {
+            return None;
+        };
+        // Committed.
+        let key = frame.stack.pop()?;
+        let container = frame.stack.pop()?;
+        let value = frame.stack.pop()?;
+        let act = self.inline_slot();
+        // SAFETY: a parked slot's locals storage is its own, and empty.
+        let locals = unsafe { &mut *act.frame.locals.as_ptr() };
+        locals.push(container);
+        locals.push(key);
+        locals.push(value);
+        fill_unbound(locals, code.varnames.len());
+        frame.pc = pc as u32 + 1;
+        let mut act = self.inline_bind(frame, shell, pc, act, code, Object::Function(f), guard);
+        act.discard = true;
+        Some(act)
+    }
+
     /// `FOR_ITER` over a generator, from the quiet loop: the generator
     /// resumes as a lean activation (see `generator_send_lean`); its
     /// value is pushed, or on exhaustion the loop exits exactly as the
@@ -11008,6 +11077,7 @@ impl Interpreter {
             let frame = unsafe { &*sw.cur };
             match frame.code.instructions.get(frame.pc as usize).map(|i| i.op) {
                 Some(OpCode::BinarySubscr) if self.core_getitem(sw, false) => return None,
+                Some(OpCode::StoreSubscr) if self.core_setitem(sw) => return None,
                 Some(OpCode::ForIter) if self.core_for_iter_lane(sw) => return None,
                 Some(OpCode::CallKw | OpCode::CallEx) if self.core_call_lane(sw) => return None,
                 Some(OpCode::BinaryOp | OpCode::SetUpdate) if self.core_set_op(sw) => return None,
@@ -16395,6 +16465,34 @@ impl Interpreter {
         true
     }
 
+    /// [`Self::core_getitem`] for a `STORE_SUBSCR` the instance's Python
+    /// `__setitem__` serves ([`Self::try_inline_setitem`]). `false`
+    /// touches nothing.
+    #[inline(never)]
+    fn core_setitem(&mut self, sw: &mut CoreSwitch) -> bool {
+        if !self.inline_calls_ok() {
+            return false;
+        }
+        let mut tmp = None;
+        // SAFETY: see `CoreSwitch` (as in `core_call`).
+        let act = unsafe {
+            let depth = (*sw.inl).len();
+            let (frame, _, shell) = sw.activation(depth, &mut tmp);
+            let pc = (*frame).pc as usize;
+            self.try_inline_setitem(&mut *frame, &mut *shell.cast::<QuietShell<'_>>(), pc)
+        };
+        let Some(mut act) = act else {
+            return false;
+        };
+        let callee: *mut Frame = &raw mut *act.frame;
+        // SAFETY: as above.
+        unsafe { push_fast(&mut *sw.inl, act) };
+        sw.cur = callee;
+        sw.scratch = usize::MAX;
+        sw.last = &raw mut sw.scratch;
+        true
+    }
+
     /// The core loop's `CALL` at `pc` of a registered leaf builtin whose
     /// leaf half declined the operands (`len(obj)` of an instance with a
     /// Python `__len__`, say): the call runs through [`Self::call`] in
diff --git a/tests/regrtest/test_setitem_python_dunder.py b/tests/regrtest/test_setitem_python_dunder.py
new file mode 100644
index 00000000..37e21514
--- /dev/null
+++ b/tests/regrtest/test_setitem_python_dunder.py
@@ -0,0 +1,136 @@
+"""`obj[key] = value` on an instance whose class defines `__setitem__`
+in Python: the method runs with the right arguments at every execution
+of a warm site, its result is discarded, exceptions carry its frame, a
+changed `__setitem__` takes effect, and replaced values die on time."""
+
+import gc
+
+
+class Store:
+    def __init__(self):
+        self.log = []
+
+    def __setitem__(self, key, value):
+        self.log.append((key, value))
+        return "ignored"
+
+
+s = Store()
+for i in range(300):
+    s[i] = i * 2
+    s["k"] = (s, i)
+assert s.log[:2] == [(0, 0), ("k", (s, 0))]
+assert len(s.log) == 600
+
+
+# A dict subclass funnelling stores through its own logic.
+class Doubling(dict):
+    def __setitem__(self, key, value):
+        dict.__setitem__(self, key, value * 2)
+
+
+d = Doubling()
+for i in range(300):
+    d[i % 7] = i
+assert d[299 % 7] == 598 and len(d) == 7
+
+
+# Exceptions propagate with the method's frame in the traceback.
+class Picky:
+    def __setitem__(self, key, value):
+        if key < 0:
+            raise KeyError(key)
+
+
+p = Picky()
+for i in range(200):
+    p[i] = i
+    try:
+        p[-i - 1] = i
+    except KeyError as e:
+        assert e.args == (-i - 1,)
+        tb = e.__traceback__
+        names = []
+        while tb is not None:
+            names.append(tb.tb_frame.f_code.co_name)
+            tb = tb.tb_next
+        assert names[-1] == "__setitem__", names
+    else:
+        raise AssertionError("expected KeyError")
+
+
+# Reassigning `__setitem__` after warm-up, including to one that takes
+# defaults or isn't a plain function.
+class Swappable:
+    def __setitem__(self, key, value):
+        self.last = ("first", key, value)
+
+
+sw = Swappable()
+
+
+def store(obj, k, v):
+    obj[k] = v
+
+
+for i in range(300):
+    store(sw, i, i)
+assert sw.last == ("first", 299, 299)
+
+
+def second(self, key, value, extra="x"):
+    self.last = ("second", key, value, extra)
+
+
+Swappable.__setitem__ = second
+for i in range(50):
+    store(sw, i, i)
+assert sw.last == ("second", 49, 49, "x")
+
+
+seen = []
+Swappable.__setitem__ = staticmethod(lambda k, v: seen.append(("static", k, v)))
+store(sw, 1, 2)
+assert seen == [("static", 1, 2)]
+del Swappable.__setitem__
+try:
+    store(sw, 1, 2)
+except TypeError as e:
+    assert "does not support item assignment" in str(e), str(e)
+else:
+    raise AssertionError("expected TypeError")
+
+
+# A replaced value dies when the store drops it, and the stored value
+# and key are released when the method is done with them.
+class Tracked:
+    alive = 0
+
+    def __init__(self):
+        Tracked.alive += 1
+
+    def __del__(self):
+        Tracked.alive -= 1
+
+
+class Drop:
+    def __setitem__(self, key, value):
+        pass
+
+
+dr = Drop()
+
+
+def drop_store():
+    dr[Tracked()] = Tracked()
+    return Tracked.alive
+
+
+gc.disable()
+try:
+    for _ in range(300):
+        assert drop_store() == 0
+finally:
+    gc.enable()
+
+print("ok")

From 0675d614824227fad81c751ebd94fbda8a03875c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 03:22:42 -0700
Subject: [PATCH 160/173] perf: keep inspect's class probes in the core loop

Three operations `inspect.signature` performs on every class or
function it examines left the core loop for the slow step:

- `sys.getrecursionlimit()` (inspect.unwrap reads it per call) was not
  among the builtin functions the core loop calls in place, though it
  reads no frame;
- `name in cls.__dict__` (membership in a class's mappingproxy) had no
  leaf form; it now tests the wrapped class dict as a dict's does;
- `weakref.ref(obj)` and `types.MappingProxyType(mapping)` calls
  (inspect's MRO cache and Signature.parameters) were not among the
  builtin types the core loop's type lane constructs through the
  ordinary call.

Measured (instructions retired, macOS, pinned hash seed), per probe
iteration:
- str(inspect.signature(dataclass)): 933k -> 920k
- class_creation probe: 3.427M -> 3.388M
---
 crates/weavepy-vm/src/lib.rs               | 13 ++++-
 tests/regrtest/test_introspection_lanes.py | 65 ++++++++++++++++++++++
 2 files changed, 75 insertions(+), 3 deletions(-)
 create mode 100644 tests/regrtest/test_introspection_lanes.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 023ffd24..c28cc7e2 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -16500,8 +16500,9 @@ impl Interpreter {
     /// nothing.
     #[inline(never)]
     /// The `CALL` at the running activation's pc of an exact builtin
-    /// container type over one positional argument (`set(genexpr)`,
-    /// `list(xs)`), through the ordinary call with the caller published
+    /// container type (or `weakref.ref`, or `types.MappingProxyType`) over
+    /// one positional argument (`set(genexpr)`, `list(xs)`), through the
+    /// ordinary call with the caller published
     /// (consuming the argument may run Python code), as the builtin lane
     /// runs a builtin function. `false` touches nothing.
     fn core_type_lane(&mut self, sw: &mut CoreSwitch) -> bool {
@@ -16528,6 +16529,9 @@ impl Interpreter {
         if ![&bt.list_, &bt.tuple_, &bt.set_, &bt.frozenset_, &bt.dict_]
             .iter()
             .any(|t| Rc::ptr_eq(t, ty))
+            // (`weakref.ref(obj)` and `types.MappingProxyType(mapping)`,
+            // as `inspect` makes them, read no frame either.)
+            && !(ty.flags.is_builtin && matches!(ty.name.as_str(), "ReferenceType" | "mappingproxy"))
         {
             return false;
         }
@@ -26547,6 +26551,9 @@ impl Interpreter {
                 }
                 found.ok().flatten()
             }
+            // A class's `__dict__` view (`'__dict__' in type.__dict__`, as
+            // `inspect` asks) tests the class dict it wraps.
+            Object::MappingProxy(d) => Self::leaf_contains(&Object::Dict(d.clone()), item),
             _ => None,
         }
     }
@@ -74163,7 +74170,7 @@ fn builtin_fn_lane_ok(name: &str) -> bool {
             | "heappushpop" | "bisect" | "bisect_left" | "bisect_right" | "insort"
             | "insort_left" | "insort_right"
             // sys
-            | "_getframe" | "_getframemodulename"
+            | "_getframe" | "_getframemodulename" | "getrecursionlimit"
             // warnings (which reads the whole stack, lean activations included)
             | "warn"
             // asyncio (`gather` reads the running loop's current task)
diff --git a/tests/regrtest/test_introspection_lanes.py b/tests/regrtest/test_introspection_lanes.py
new file mode 100644
index 00000000..6b83d944
--- /dev/null
+++ b/tests/regrtest/test_introspection_lanes.py
@@ -0,0 +1,65 @@
+"""Operations `inspect` leans on, at warm sites: membership in a class's
+`__dict__` view, `sys.getrecursionlimit()`, and constructing `weakref.ref`
+and `types.MappingProxyType` objects."""
+
+import sys
+import types
+import weakref
+
+
+class A:
+    x = 1
+
+
+for i in range(300):
+    assert "x" in A.__dict__
+    assert "missing" not in A.__dict__
+    assert ("x" not in A.__dict__) is False
+    assert "__dict__" in A.__dict__
+    assert "__mro__" in type.__dict__
+A.y = 2
+assert "y" in A.__dict__
+del A.y
+assert "y" not in A.__dict__
+
+old = sys.getrecursionlimit()
+try:
+    for limit in (old, old + 50, old + 100):
+        sys.setrecursionlimit(limit)
+        for _ in range(100):
+            assert sys.getrecursionlimit() == limit
+finally:
+    sys.setrecursionlimit(old)
+
+
+class Target:
+    pass
+
+
+for _ in range(300):
+    t = Target()
+    r = weakref.ref(t)
+    assert r() is t
+    del t
+    assert r() is None
+
+for i in range(300):
+    d = {"k": i}
+    proxy = types.MappingProxyType(d)
+    assert proxy["k"] == i and len(proxy) == 1
+    d["k2"] = 0
+    assert "k2" in proxy
+try:
+    types.MappingProxyType(1)
+except TypeError as e:
+    assert "mappingproxy() argument must be a mapping" in str(e), str(e)
+else:
+    raise AssertionError("expected TypeError")
+try:
+    weakref.ref(1)
+except TypeError as e:
+    assert "cannot create weak reference to 'int' object" in str(e), str(e)
+else:
+    raise AssertionError("expected TypeError")
+
+print("ok")

From b2c19035810a51b9ab919ecf170f1cd1d8af54fd Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 03:38:04 -0700
Subject: [PATCH 161/173] perf: settle `in` over IntEnum members and functions
 without the slow step

The core loop's membership test over a tuple or list (leaf_contains)
compared each element natively only for identical objects, native
scalars and plain instances; any other pair sent the whole test to the
slow step's rich comparison. Two common shapes now settle in place, for
`in` only (list.remove's admission check keeps the narrower rule, as
its native removal compares on its own):

- an instance of an `int` subclass whose `__eq__` is int's own (an
  IntEnum or IntFlag member: `kind in (_POSITIONAL_ONLY, ...)` in
  inspect, enum's own checks) compares by its value against ints, bools
  and others of its kind;
- a function, or a class under type's own `__eq__`, equals only itself
  against scalars, builtin functions, plain instances and its own kind.

Any user `__eq__`, on either side or on a metaclass, still runs through
the full comparison.

Measured (instructions retired, macOS, interpreter only):
- Parameter.POSITIONAL_OR_KEYWORD in (two other kinds): 6776 -> 929
- f in (len, None, f): 4625 -> 1460
- str(inspect.signature(dataclass)): 920k -> 869k per call
- class_creation probe: 3.388M -> 3.316M per iteration
---
 crates/weavepy-vm/src/lib.rs                  | 64 +++++++++++-
 ...test_contains_int_subclass_and_identity.py | 99 +++++++++++++++++++
 2 files changed, 162 insertions(+), 1 deletion(-)
 create mode 100644 tests/regrtest/test_contains_int_subclass_and_identity.py

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index c28cc7e2..f345b176 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -26491,7 +26491,7 @@ impl Interpreter {
         if let Some(found) = Self::leaf_contains_table(container, item) {
             return Some(found);
         }
-        let member = Self::leaf_member_eq;
+        let member = Self::leaf_contains_eq;
         match container {
             Object::List(l) => {
                 // SAFETY: a read between two instructions (the member test
@@ -26656,6 +26656,68 @@ impl Interpreter {
         None
     }
 
+    /// [`Self::leaf_member_eq`] for a membership test (`in`), which settles
+    /// a few more pairs itself rather than through a native comparison.
+    fn leaf_contains_eq(x: &Object, item: &Object) -> Option<bool> {
+        if let Some(eq) = Self::leaf_member_eq(x, item) {
+            return Some(eq);
+        }
+        let scalar = |o: &Object| {
+            matches!(
+                o,
+                Object::None
+                    | Object::Bool(_)
+                    | Object::Int(_)
+                    | Object::Float(_)
+                    | Object::Str(_)
+                    | Object::Bytes(_)
+            )
+        };
+        // An `int` subclass's instance under `int`'s own `__eq__` (an
+        // `IntEnum` or `IntFlag` member) equals what its value equals.
+        if let (Some(a), Some(b)) = (Self::int_eq_value(x), Self::int_eq_value(item)) {
+            return Some(a == b);
+        }
+        // A function, or a class under `type`'s own `__eq__`, equals
+        // nothing but itself, against anything whose `__eq__` doesn't
+        // know it either.
+        let identity = |o: &Object| match o {
+            Object::Function(_) => true,
+            Object::Type(t) => Self::plain_metaclass(t),
+            _ => false,
+        };
+        // (A builtin function's `__eq__` knows only builtin functions.)
+        let default_eq = |o: &Object| {
+            identity(o)
+                || scalar(o)
+                || matches!(o, Object::Builtin(_))
+                || (matches!(o, Object::Instance(i) if i.native.get().is_none())
+                    && !crate::object::instance_has_custom_eq(o))
+        };
+        if (identity(x) && default_eq(item)) || (identity(item) && default_eq(x)) {
+            return Some(false);
+        }
+        None
+    }
+
+    /// The value `o` compares as under `int.__eq__`: a machine `int` or
+    /// `bool`, or an instance of an `int` subclass whose `__eq__` is
+    /// `int`'s own, over a machine `int`.
+    fn int_eq_value(o: &Object) -> Option<i64> {
+        match o {
+            Object::Int(i) => Some(*i),
+            Object::Bool(b) => Some(i64::from(*b)),
+            Object::Instance(inst) => {
+                let Some(Object::Int(i)) = inst.native.get() else {
+                    return None;
+                };
+                let eq = inst.cls_raw().dunder(crate::types::Dunder::Eq);
+                (eq.builtin_owner() && !eq.object_owner()).then_some(*i)
+            }
+            _ => None,
+        }
+    }
+
     /// `len(v)` for an instance whose class serves `__len__` with a
     /// registered native leaf builtin (found through the class's leaf
     /// attribute cache): the length, or `None` for the full path (which
diff --git a/tests/regrtest/test_contains_int_subclass_and_identity.py b/tests/regrtest/test_contains_int_subclass_and_identity.py
new file mode 100644
index 00000000..38706396
--- /dev/null
+++ b/tests/regrtest/test_contains_int_subclass_and_identity.py
@@ -0,0 +1,99 @@
+"""`x in (a, b)` with `int`-subclass members (IntEnum, IntFlag) compares
+by value under `int.__eq__`, functions and classes by identity, and any
+user `__eq__` (on either side, or on a metaclass) still runs."""
+
+import enum
+
+
+class Color(enum.IntEnum):
+    RED = 1
+    GREEN = 2
+
+
+class Perm(enum.IntFlag):
+    R = 4
+    W = 2
+
+
+for _ in range(300):
+    assert Color.RED in (Color.GREEN, Color.RED)
+    assert Color.RED in (2, 1)
+    assert 2 in (Color.RED, Color.GREEN)
+    assert True in (Color.RED,)
+    assert Color.RED not in (Color.GREEN, 3)
+    assert Perm.R in (Perm.W, 4)
+    assert (Perm.R | Perm.W) in (6,)
+    assert 1.0 in (Color.RED,)
+    assert Color.GREEN in [0, 2.0]
+
+
+class Loud(int):
+    calls = 0
+
+    def __eq__(self, other):
+        Loud.calls += 1
+        return True
+
+    __hash__ = int.__hash__
+
+
+for _ in range(100):
+    assert 5 in (Loud(1),)
+    assert Loud(1) in (7,)
+assert Loud.calls == 200
+
+
+def f():
+    pass
+
+
+def g():
+    pass
+
+
+class K:
+    pass
+
+
+for _ in range(300):
+    assert f in (None, g, f)
+    assert f not in (None, g, "f", 1, K)
+    assert K in (int, K)
+    assert K not in (int, f, None)
+    assert f in (len, None, f)
+    assert f not in (len, print)
+    assert len in (f, len)
+    assert K not in (len,)
+
+
+class Wild:
+    calls = 0
+
+    def __eq__(self, other):
+        Wild.calls += 1
+        return other is f
+
+
+w = Wild()
+for _ in range(100):
+    assert f in (w,)
+    assert K not in (w,)
+assert Wild.calls == 200
+
+
+class Meta(type):
+    def __eq__(cls, other):
+        return other == "anything"
+
+    __hash__ = type.__hash__
+
+
+class M(metaclass=Meta):
+    pass
+
+
+for _ in range(100):
+    assert "anything" in (M,)
+    assert M in ("anything",)
+
+print("ok")

From ca9f55211ae5ea0668dc7cb006415c9feb4be773 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 11 Oct 2026 03:45:29 -0700
Subject: [PATCH 162/173] perf: read a class __dict__ view's present keys in
 the core loop

`type.__dict__['__mro__']` (inspect's _check_class reads type's own
getsets this way) subscripted a mappingproxy, which the core loop's
container op did not take, so it left for the slow step. A str key
present in the wrapped class dict is now read as a dict's is; a miss
still raises its KeyError on the full path.

Measured (instructions retired, macOS, pinned hash seed):
str(inspect.signature(dataclass)) 869k -> 866k per call.
---
 crates/weavepy-vm/src/lib.rs               | 15 ++++++++++++++-
 tests/regrtest/test_introspection_lanes.py | 15 ++++++++++++---
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index f345b176..76e825d7 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -18683,12 +18683,25 @@ impl Interpreter {
                 // grades its own release.)
                 if !matches!(
                     c,
-                    Object::List(_) | Object::Tuple(_) | Object::Dict(_) | Object::Str(_)
+                    Object::List(_)
+                        | Object::Tuple(_)
+                        | Object::Dict(_)
+                        | Object::Str(_)
+                        | Object::MappingProxy(_)
                 ) || !Self::core_droppable(c)
                 {
                     return None;
                 }
                 let r = match (c, k) {
+                    // A class's `__dict__` view (`type.__dict__['__mro__']`,
+                    // as `inspect` reads it): a present key's value (a miss
+                    // raises on the full path).
+                    (Object::MappingProxy(d), Object::Str(_)) => {
+                        let probe = crate::object::LeafProbe::new(k)?;
+                        // SAFETY: as for a dict below.
+                        let d = unsafe { d.peek() }?;
+                        clone_hot(d.get_hot(&probe)?)
+                    }
                     (Object::List(xs), Object::Int(i)) => {
                         // SAFETY: a read between two instructions.
                         let xs = unsafe { xs.peek() }?;
diff --git a/tests/regrtest/test_introspection_lanes.py b/tests/regrtest/test_introspection_lanes.py
index 6b83d944..06dd3cb2 100644
--- a/tests/regrtest/test_introspection_lanes.py
+++ b/tests/regrtest/test_introspection_lanes.py
@@ -1,5 +1,5 @@
-"""Operations `inspect` leans on, at warm sites: membership in a class's
-`__dict__` view, `sys.getrecursionlimit()`, and constructing `weakref.ref`
+"""Operations `inspect` leans on, at warm sites: membership in and
+subscripts of a class's `__dict__` view, `sys.getrecursionlimit()`, and constructing `weakref.ref`
 and `types.MappingProxyType` objects."""
 
 import sys
@@ -17,8 +17,17 @@ class A:
     assert ("x" not in A.__dict__) is False
     assert "__dict__" in A.__dict__
     assert "__mro__" in type.__dict__
+for i in range(300):
+    assert A.__dict__["x"] == 1
+    assert type.__dict__["__mro__"].__get__(A) == (A, object)
+    try:
+        A.__dict__["missing"]
+    except KeyError as e:
+        assert e.args == ("missing",)
+    else:
+        raise AssertionError("expected KeyError")
 A.y = 2
-assert "y" in A.__dict__
+assert "y" in A.__dict__ and A.__dict__["y"] == 2
 del A.y
 assert "y" not in A.__dict__
 

From ba765108c295c12afeee1f93ccabb82332dc1370 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 18 Oct 2026 22:36:00 -0700
Subject: [PATCH 163/173] fix: compile the posixpath accelerators only where
 they're registered

os._path_splitroot_ex and os._path_normpath are registered on non-Windows
targets only, but their helpers were compiled everywhere, so the Windows
build failed on seven dead-code warnings under -D warnings. The helpers
now carry the same #[cfg(not(windows))] as the registration.
---
 crates/weavepy-vm/src/stdlib/os.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/crates/weavepy-vm/src/stdlib/os.rs b/crates/weavepy-vm/src/stdlib/os.rs
index 72c49fe1..6263f082 100644
--- a/crates/weavepy-vm/src/stdlib/os.rs
+++ b/crates/weavepy-vm/src/stdlib/os.rs
@@ -7763,6 +7763,7 @@ pub(crate) fn static_name_key(name: &'static str) -> DictKey {
 /// The length of a POSIX path's root (`posixpath.splitroot`'s middle part):
 /// none for a relative path, one separator for an absolute one, and two
 /// for exactly two leading separators (implementation defined by POSIX).
+#[cfg(not(windows))]
 fn posix_root_len<T: PartialEq>(p: &[T], sep: &T) -> usize {
     if p.first() != Some(sep) {
         0
@@ -7776,6 +7777,7 @@ fn posix_root_len<T: PartialEq>(p: &[T], sep: &T) -> usize {
 /// `posixpath.normpath`'s pure-Python algorithm over the units of `path`
 /// (bytes, UTF-8 bytes or code points): `A//B`, `A/./B` and `A/foo/../B`
 /// all become `A/B`.
+#[cfg(not(windows))]
 fn posix_normpath_units<T: PartialEq + Copy>(path: &[T], sep: T, dot: T) -> Vec<T> {
     if path.is_empty() {
         return vec![dot];
@@ -7813,6 +7815,7 @@ fn posix_normpath_units<T: PartialEq + Copy>(path: &[T], sep: T, dot: T) -> Vec<
 /// A path argument of `func` as `str` or `bytes` (an `os.PathLike`'s
 /// `__fspath__()`), with CPython's `path_t` converter's message for
 /// anything else.
+#[cfg(not(windows))]
 fn path_arg(func: &str, p: &Object) -> Result<Object, RuntimeError> {
     if matches!(p, Object::Instance(_)) {
         return os_fspath(std::slice::from_ref(p));
@@ -7827,6 +7830,7 @@ fn path_arg(func: &str, p: &Object) -> Result<Object, RuntimeError> {
 }
 
 /// The leaf half of `_path_splitroot_ex` for a `str` or `bytes` path.
+#[cfg(not(windows))]
 fn path_splitroot_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
     let r = match args {
         [p @ Object::Str(s)] => {
@@ -7862,6 +7866,7 @@ fn path_splitroot_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>>
 
 /// `posix._path_splitroot_ex(p)`: split a pathname into drive (always
 /// empty on POSIX), root and tail, as `posixpath`'s fallback does.
+#[cfg(not(windows))]
 fn os_path_splitroot_ex(args: &[Object]) -> Result<Object, RuntimeError> {
     let [p] = args else {
         return Err(type_error(format!(
@@ -7890,6 +7895,7 @@ fn os_path_splitroot_ex(args: &[Object]) -> Result<Object, RuntimeError> {
 }
 
 /// The leaf half of `_path_normpath` for a `str` or `bytes` path.
+#[cfg(not(windows))]
 fn path_normpath_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
     let r = match args {
         [Object::Str(s)] => {
@@ -7909,6 +7915,7 @@ fn path_normpath_fast(args: &[Object]) -> Option<Result<Object, RuntimeError>> {
 
 /// `posix._path_normpath(path)`: `posixpath.normpath`, as its pure-Python
 /// fallback computes it.
+#[cfg(not(windows))]
 fn os_path_normpath(args: &[Object]) -> Result<Object, RuntimeError> {
     let [p] = args else {
         return Err(type_error(format!(

From 1131f92bea56e94e009f6cef0ef14961ae77af2c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 18 Oct 2026 22:36:00 -0700
Subject: [PATCH 164/173] style: satisfy clippy and rustfmt

Clippy (-D warnings) and rustfmt found issues across the merged
performance work: elidable lifetimes, raw-pointer borrows and casts,
index loops over parallel slices, a match on a bool, a redundant else
in the core loop's offload macro, and a &Option return. The boxed
ChainRec pool and the frame JIT's method caches keep their boxes (their
contents are addressed by pointer) with the lint allowed and the reason
noted, as the neighbouring cache vectors already do; the LRU order
buffer's unaligned word reads and writes allow the alignment lint.

rustfmt's check had been crashing on CI (an out-of-memory panic while
diffing a large file) instead of reporting these diffs.
---
 crates/weavepy-vm/src/dictmap.rs              |  40 +++--
 crates/weavepy-vm/src/frame_jit.rs            |  80 ++++++---
 crates/weavepy-vm/src/gc_trace.rs             |   2 +-
 crates/weavepy-vm/src/gen_fast.rs             |   3 +-
 crates/weavepy-vm/src/inst_dict.rs            |  10 +-
 crates/weavepy-vm/src/lib.rs                  | 159 +++++++++---------
 crates/weavepy-vm/src/object.rs               |   4 +-
 .../weavepy-vm/src/stdlib/asyncio_events.rs   |  15 +-
 .../weavepy-vm/src/stdlib/faulthandler_mod.rs |   6 +-
 crates/weavepy-vm/src/stdlib/functools_mod.rs |  24 +--
 .../src/stdlib/functools_mod/lru_order.rs     |  14 +-
 crates/weavepy-vm/src/stdlib/pickle_accel.rs  |   1 -
 .../src/stdlib/pickle_accel/classes.rs        |   1 -
 crates/weavepy-vm/src/stdlib/sys.rs           |  14 +-
 crates/weavepy-vm/src/tier2.rs                |   4 +-
 crates/weavepy-vm/src/types.rs                |  34 ++--
 16 files changed, 224 insertions(+), 187 deletions(-)

diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index bc55c0ca..c9b1198e 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -79,10 +79,14 @@ impl Probe for DictKey {
         // A new string (its hash not yet computed) compares with the
         // stored strings instead.
         match (&self.0, &stored.0) {
-            (Object::Str(a), Object::Str(b)) if crate::shared_value::SharedStr::known_hash(a).is_none() => {
+            (Object::Str(a), Object::Str(b))
+                if crate::shared_value::SharedStr::known_hash(a).is_none() =>
+            {
                 Some(a.as_bytes() == b.as_bytes())
             }
-            (Object::Str(a), Object::WStr(_)) if crate::shared_value::SharedStr::known_hash(a).is_none() => {
+            (Object::Str(a), Object::WStr(_))
+                if crate::shared_value::SharedStr::known_hash(a).is_none() =>
+            {
                 Some(false)
             }
             _ => None,
@@ -279,7 +283,10 @@ impl<V> Table<V> {
 
     /// [`Self::reserve`], reporting an allocation failure instead of
     /// aborting.
-    pub fn try_reserve(&mut self, additional: usize) -> Result<(), std::collections::TryReserveError> {
+    pub fn try_reserve(
+        &mut self,
+        additional: usize,
+    ) -> Result<(), std::collections::TryReserveError> {
         self.entries.try_reserve(additional)?;
         self.reserve_slots(additional);
         Ok(())
@@ -450,12 +457,9 @@ impl<V> Table<V> {
     fn scan_unhashed<Q: Probe + ?Sized>(&self, key: &Q) -> Option<Option<usize>> {
         let mut found = None;
         for (i, b) in self.entries.iter().enumerate() {
-            match key.probe_eq_unhashed(&b.key)? {
-                true => {
-                    found = Some(i);
-                    break;
-                }
-                false => {}
+            if key.probe_eq_unhashed(&b.key)? {
+                found = Some(i);
+                break;
             }
         }
         Some(found)
@@ -547,7 +551,7 @@ impl<V> Table<V> {
     pub fn keys_with_hash(&self, hash: i64) -> Vec<&DictKey> {
         let mut out = Vec::new();
         let _ = self.find_by_hash(hash, |k| {
-            out.push(k as *const DictKey);
+            out.push(std::ptr::from_ref::<DictKey>(k));
             false
         });
         // SAFETY: the pointers address entries of `self`, borrowed for the
@@ -923,7 +927,10 @@ impl<V> Table<V> {
     }
 
     /// Sort the entries by key with `cmp` (stable).
-    pub fn sort_by_key_with(&mut self, mut cmp: impl FnMut(&DictKey, &DictKey) -> std::cmp::Ordering) {
+    pub fn sort_by_key_with(
+        &mut self,
+        mut cmp: impl FnMut(&DictKey, &DictKey) -> std::cmp::Ordering,
+    ) {
         self.entries.sort_by(|a, b| cmp(&a.key, &b.key));
         self.reindex();
     }
@@ -1087,7 +1094,7 @@ impl<'a, V> VacantEntry<'a, V> {
     }
 }
 
-impl<'a, V> ProbeVacancy<'a, V> {
+impl<V> ProbeVacancy<'_, V> {
     /// The probe's Python hash.
     #[inline]
     pub fn hash(&self) -> i64 {
@@ -1755,7 +1762,10 @@ mod tests {
         assert!(!m.slots.is_empty());
         let c = m.clone();
         check(&c);
-        assert_eq!(c.get(&k(5)).map(|o| matches!(o, Object::Int(5))), Some(true));
+        assert_eq!(
+            c.get(&k(5)).map(|o| matches!(o, Object::Int(5))),
+            Some(true)
+        );
         assert!(c.get(&k(50)).is_none());
     }
 
@@ -1779,7 +1789,9 @@ mod tests {
                 assert_eq!(m.len(), n as usize);
                 check(&m);
             }
-            assert!(m.replace_index_unique(n as usize, 0, k(-1), Object::None).is_none());
+            assert!(m
+                .replace_index_unique(n as usize, 0, k(-1), Object::None)
+                .is_none());
         }
     }
 }
diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 34624412..7ee92a21 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -291,7 +291,9 @@ impl Native {
             } else {
                 ""
             };
-            stats::note(&self.name, &self.ops, from, st.pc, st.last, st.sw, status, bail);
+            stats::note(
+                &self.name, &self.ops, from, st.pc, st.last, st.sw, status, bail,
+            );
         }
     }
 
@@ -321,7 +323,9 @@ impl Native {
             } else {
                 ""
             };
-            stats::note(&self.name, &self.ops, from, st.pc, st.last, st.sw, status, bail);
+            stats::note(
+                &self.name, &self.ops, from, st.pc, st.last, st.sw, status, bail,
+            );
         }
         status
     }
@@ -885,15 +889,20 @@ mod stats {
             // (Where it switched: the last instruction it ran, and the
             // activation it switched to.)
             (super::RELOAD, _) => {
-                // SAFETY: a live switch's running activation.
-                let to = (!sw.is_null())
-                    .then(|| unsafe {
+                let to = if sw.is_null() {
+                    weavepy_compiler::Name::default()
+                } else {
+                    // SAFETY: a live switch's running activation.
+                    unsafe {
                         let cur: &crate::Frame = &*(*sw).cur;
                         cur.code.qualname.clone()
-                    })
-                    .unwrap_or_default();
+                    }
+                };
                 match ops.get(last) {
-                    Some(op) => format!("<switched> at {op:?}@{last}, {:?} to {to}", ops.get(last + 1)),
+                    Some(op) => format!(
+                        "<switched> at {op:?}@{last}, {:?} to {to}",
+                        ops.get(last + 1)
+                    ),
                     None => "<switched>".to_owned(),
                 }
             }
@@ -1375,7 +1384,11 @@ unsafe extern "C" fn h_for_iter_rest(
                 // A collapsed chain's fast step delivered the value without
                 // a switch (see `Interpreter::chain_step`).
                 if std::ptr::eq(sw.cur, st.frame) {
-                    return if sw.pending.is_none() { 1 } else { FOR_SWITCHED };
+                    return if sw.pending.is_none() {
+                        1
+                    } else {
+                        FOR_SWITCHED
+                    };
                 }
                 // A compiled body runs from here; its yield (the value on
                 // top, past the instruction) or its return (the iterator
@@ -2125,10 +2138,10 @@ unsafe fn two_strs<'s>(
 /// # Safety
 ///
 /// As [`Kernel`]'s.
-unsafe fn ascii_from<'s>(
-    s: &'s crate::shared_value::SharedStr,
+unsafe fn ascii_from(
+    s: &crate::shared_value::SharedStr,
     start: *const Object,
-) -> Option<Option<(&'s [u8], usize)>> {
+) -> Option<Option<(&[u8], usize)>> {
     let bytes = s.as_bytes();
     if start.is_null() {
         return Some(Some((bytes, 0)));
@@ -2325,7 +2338,11 @@ unsafe extern "C" fn k_str_format(
             v.is_null()
                 || matches!(
                     &*v,
-                    Object::Str(_) | Object::Int(_) | Object::Bool(_) | Object::Float(_) | Object::None
+                    Object::Str(_)
+                        | Object::Int(_)
+                        | Object::Bool(_)
+                        | Object::Float(_)
+                        | Object::None
                 )
         };
         if !(plain(a) && plain(b)) {
@@ -3700,7 +3717,7 @@ impl ModCache {
         };
         // SAFETY: a read between two instructions.
         if let Some(d) = unsafe { m.dict.peek() } {
-            self.dict.set(Rc::as_ptr(&m.dict) as *const u8 as usize);
+            self.dict.set(Rc::as_ptr(&m.dict).cast::<u8>() as usize);
             self.stamp.set(d.mutation_stamp());
             self.idx.set(key_idx as usize);
         }
@@ -3882,7 +3899,9 @@ unsafe extern "C" fn h_stack_attr(
                 return 0;
             }
             // A function's stored attribute (`wrapper.calls`).
-            Object::Function(_) => return u32::from(!Interpreter::core_function_load(code, arg, at)),
+            Object::Function(_) => {
+                return u32::from(!Interpreter::core_function_load(code, arg, at))
+            }
             _ => None,
         };
         let Some(v) = v else {
@@ -5104,8 +5123,7 @@ unsafe fn direct_run<const PUSHED: bool>(
                     a.act.shell.is_none()
                         && (!PUSHED
                             || ((cframe.code.cellvars.is_empty() || a.owns_cells)
-                                && (a.init_inst.is_none()
-                                    || matches!(&*cst.stack, Object::None))))
+                                && (a.init_inst.is_none() || matches!(&*cst.stack, Object::None))))
                 })
                 && Rc::strong_count(&cframe.locals) == 1
                 && !interp.countdown_out(2)
@@ -5778,7 +5796,8 @@ unsafe extern "C" fn h_call_leaf(
         let lenu = len as usize;
         let start = lenu - site.argc - 2;
         let base = s.stack;
-        let hit = matches!(&*base.add(start), Object::Builtin(b) if std::ptr::eq(Rc::as_ptr(b), site.f));
+        let hit =
+            matches!(&*base.add(start), Object::Builtin(b) if std::ptr::eq(Rc::as_ptr(b), site.f));
         if hit && !s.sw.is_null() {
             let first = if matches!(&*base.add(start + 1), Object::Unbound) {
                 start + 2
@@ -6221,7 +6240,11 @@ unsafe extern "C" fn h_call_ex(st: *mut State, code: *const CodeObject, pc: u64,
             if sw.pending.is_none() && !interp.core_call_lane(sw) {
                 sw.pending = Some(crate::CoreExit::Stop(crate::LeafStop::Step));
             }
-            return if finished_in_place(st, start) { 0 } else { RELOAD };
+            return if finished_in_place(st, start) {
+                0
+            } else {
+                RELOAD
+            };
         }
         if sw.pending.is_none() && !std::ptr::eq(sw.cur, st.frame) {
             if let Some(r) = run_pushed_directly(st, pc, start, saved_last) {
@@ -6400,7 +6423,9 @@ unsafe extern "C" fn h_send(st: *mut State, pc: u64, len: u64) -> u32 {
         }
         // A simple generator steps to its next yield in place, as the core
         // loop's arm steps it (see `gen_fast`).
-        if let (Object::Generator(g), Object::None) = (&*st.stack.add(len - 2), &*st.stack.add(len - 1)) {
+        if let (Object::Generator(g), Object::None) =
+            (&*st.stack.add(len - 2), &*st.stack.add(len - 1))
+        {
             if crate::gen_fast::may_step(g) {
                 if let Some(r) = h_send_step(st, pc, len) {
                     return r;
@@ -7268,6 +7293,7 @@ struct Lower<'a> {
     global_caches: Vec<Box<GlobalCache>>,
     #[allow(clippy::vec_box)]
     slot_caches: Vec<Box<SlotCache>>,
+    #[allow(clippy::vec_box)]
     method_caches: Vec<Box<MethodCache>>,
     #[allow(clippy::vec_box)]
     direct_sites: Vec<Box<DirectSite>>,
@@ -8925,15 +8951,15 @@ impl<'a> Lower<'a> {
     fn native_body_call(&self, pc: usize) -> bool {
         let ins = &self.code.instructions;
         let mut n = 0;
-        for k in pc..ins.len().min(pc + 16) {
-            match ins[k].op {
+        for (k, i) in ins.iter().enumerate().skip(pc).take(16) {
+            match i.op {
                 OpCode::LoadFast
                 | OpCode::LoadFastBorrow
                 | OpCode::LoadConst
                 | OpCode::LoadSmallInt
                 | OpCode::BinaryOp => n += 1,
                 OpCode::Call => {
-                    return n >= ins[k].arg as usize
+                    return n >= i.arg as usize
                         && matches!(crate::call_site_builtin(self.ext, k), Some((_, None, _)));
                 }
                 _ => return false,
@@ -10706,7 +10732,10 @@ impl<'a> Lower<'a> {
                     self.branch_out(bad, miss);
                     let clayout = self.b.ins().load(ptr, FLAGS, cls, type_slot_layout(&l));
                     let borrow = self.b.ins().sload32(FLAGS, inst, l.inst_slots_borrow);
-                    let form = self.b.ins().uload8(types::I32, FLAGS, inst, l.inst_slots_tag);
+                    let form = self
+                        .b
+                        .ins()
+                        .uload8(types::I32, FLAGS, inst, l.inst_slots_tag);
                     let names = self.b.ins().load(ptr, FLAGS, inst, l.inst_slots_layout);
                     let vals = self.b.ins().load(ptr, FLAGS, inst, l.inst_slots_values);
                     let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0);
@@ -11698,7 +11727,8 @@ impl<'a> Lower<'a> {
 /// from its pointer (measured as `layout`'s `type_attr_version` is).
 fn type_slot_layout(l: &weavepy_jit::ObjLayout) -> i32 {
     use crate::types::TypeObject;
-    let base = l.type_attr_version as isize - std::mem::offset_of!(TypeObject, attr_version) as isize;
+    let base =
+        l.type_attr_version as isize - std::mem::offset_of!(TypeObject, attr_version) as isize;
     (base + std::mem::offset_of!(TypeObject, slot_layout_word) as isize) as i32
 }
 
diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs
index 1d4f8afc..e2ee8dff 100644
--- a/crates/weavepy-vm/src/gc_trace.rs
+++ b/crates/weavepy-vm/src/gc_trace.rs
@@ -1949,7 +1949,7 @@ impl GcState {
             }
         }
         if stats_debug() {
-            let mut kinds: std::collections::BTreeMap<String, usize> = Default::default();
+            let mut kinds = std::collections::BTreeMap::<String, usize>::default();
             for t in &temps {
                 *kinds.entry(t.obj.type_name_owned()).or_default() += 1;
             }
diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs
index f9431979..1769ce73 100644
--- a/crates/weavepy-vm/src/gen_fast.rs
+++ b/crates/weavepy-vm/src/gen_fast.rs
@@ -1084,8 +1084,7 @@ impl Interpreter {
         // (which resumes a parked chain without settling it), and a body no
         // fast step runs declines at once (its code's verdict is cached:
         // see `code_ok`).
-        if g.chain.get() != 0 || matches!(&g.code, Object::Code(code) if code_ok(code) == CODE_NO)
-        {
+        if g.chain.get() != 0 || matches!(&g.code, Object::Code(code) if code_ok(code) == CODE_NO) {
             return GenNext::Declined;
         }
         if depth > MAX_DEPTH
diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs
index d15e4760..4980859c 100644
--- a/crates/weavepy-vm/src/inst_dict.rs
+++ b/crates/weavepy-vm/src/inst_dict.rs
@@ -740,12 +740,7 @@ impl SplitValues {
     /// that position (the name there, in case the table's address was
     /// reused); anything else searches and remembers.
     #[inline]
-    pub fn get_memo(
-        &self,
-        name: &str,
-        hash: i64,
-        memo: &PosMemo,
-    ) -> Option<&Object> {
+    pub fn get_memo(&self, name: &str, hash: i64, memo: &PosMemo) -> Option<&Object> {
         let keys = self.keys()?;
         let n = self.len();
         let at = std::ptr::from_ref(keys) as usize;
@@ -759,8 +754,7 @@ impl SplitValues {
             let h = unsafe { *keys.hashes[i].get() };
             if h == hash
                 && keys.get(i).is_some_and(|key| {
-                    key_text(key) == memo.text.load(Ordering::Relaxed)
-                        || key_names_str(key, name)
+                    key_text(key) == memo.text.load(Ordering::Relaxed) || key_names_str(key, name)
                 })
             {
                 return self.values().get(i);
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 76e825d7..724caaa8 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -1008,7 +1008,9 @@ pub struct Interpreter {
     /// ever dereferenced on the thread that pushed them, while the
     /// activations they name are live.
     lean_pending: Vec<usize>,
-    /// Spare [`ChainRec`]s (empty).
+    /// Spare [`ChainRec`]s (empty), boxed: a parked chain's levels point
+    /// at their record.
+    #[allow(clippy::vec_box)]
     chain_pool: Vec<Box<ChainRec>>,
     /// The builtins the leaf burst may call directly (see [`LeafFns`]),
     /// resolved on first use.
@@ -3338,11 +3340,7 @@ impl Interpreter {
                     Object::Instance(i) => Object::Type(i.cls()),
                     _ => Object::None,
                 };
-                (
-                    inst,
-                    ty,
-                    pyexc.traceback_entries(),
-                )
+                (inst, ty, pyexc.traceback_entries())
             }
             other => {
                 let inst = crate::builtin_types::make_exception("RuntimeError", other.to_string());
@@ -11658,9 +11656,8 @@ impl Interpreter {
                         pc = at.pc;
                         last = at.last;
                         continue;
-                    } else {
-                        break Some(CoreExit::Reload);
                     }
+                    break Some(CoreExit::Reload);
                 }};
             }
             // The native code writes the stack at fixed offsets: a stack
@@ -15901,8 +15898,7 @@ impl Interpreter {
             let act = self.inline_slot();
             // SAFETY: a parked slot's locals storage is its own, and empty.
             let locals = &mut *act.frame.locals.as_ptr();
-            let callable =
-                Self::lean_call_fill(frame, &code, true, n - 4, n - 5, missing, locals);
+            let callable = Self::lean_call_fill(frame, &code, true, n - 4, n - 5, missing, locals);
             frame.pc = pc as u32 + 1;
             self.inline_bind(frame, shell, pc, act, code, callable, guard)
         };
@@ -16041,7 +16037,16 @@ impl Interpreter {
         // SAFETY: as above; the checks above hold.
         unsafe {
             self.core_bind_plain::<false>(
-                sw, pc, callee_slot, has_self, missing, false, fp, act, guard, false,
+                sw,
+                pc,
+                callee_slot,
+                has_self,
+                missing,
+                false,
+                fp,
+                act,
+                guard,
+                false,
             )
         };
         true
@@ -16130,7 +16135,9 @@ impl Interpreter {
         };
         // SAFETY: as above.
         unsafe {
-            self.core_bind_act::<DIRECT>(sw, frame, pc, fp, code_rc, act, guard, held, cells, callable)
+            self.core_bind_act::<DIRECT>(
+                sw, frame, pc, fp, code_rc, act, guard, held, cells, callable,
+            )
         }
     }
 
@@ -16185,7 +16192,9 @@ impl Interpreter {
                 frame.stack.set_len(n - 1);
             }
             let callable = frame.stack.pop().expect("the callee");
-            self.core_bind_act::<true>(sw, frame, pc, fp, code_rc, act, guard, false, false, callable)
+            self.core_bind_act::<true>(
+                sw, frame, pc, fp, code_rc, act, guard, false, false, callable,
+            )
         }
     }
 
@@ -18169,8 +18178,8 @@ impl Interpreter {
         left[..shape.uses.len()].copy_from_slice(&shape.uses);
         let defaults = &init.defaults[init.defaults.len() - missing..];
         // A split instance's values, all computed, go down in one pass.
-        let mut split_vals =
-            [const { std::mem::MaybeUninit::<Object>::uninit() }; crate::inst_dict::SHARED_KEYS_CAP];
+        let mut split_vals = [const { std::mem::MaybeUninit::<Object>::uninit() };
+            crate::inst_dict::SHARED_KEYS_CAP];
         for (k, &(_, src)) in shape.stores.iter().enumerate() {
             let value = match src {
                 StoreSrc::Param(0) => inst.clone(),
@@ -18208,19 +18217,19 @@ impl Interpreter {
             // laid out over its class's layout at allocation.
             match unsafe { i.slots.peek_mut() }.and_then(|s| s.values_for_layout_mut(layout)) {
                 Some(places) if members[..n].iter().all(|&m| usize::from(m) < places.len()) => {
-                    for k in 0..n {
+                    for (v, &m) in vals[..n].iter().zip(&members[..n]) {
                         // SAFETY: as above.
-                        let value = unsafe { std::ptr::read(&vals[k]) };
-                        drop(std::mem::replace(&mut places[usize::from(members[k])], value));
+                        let value = unsafe { std::ptr::read(v) };
+                        drop(std::mem::replace(&mut places[usize::from(m)], value));
                     }
                 }
                 _ => {
                     // (Unreachable for a fresh instance; the plain stores
                     // keep it correct regardless.)
-                    for k in 0..n {
+                    for (v, store) in vals[..n].iter().zip(&shape.stores[..n]) {
                         // SAFETY: as above.
-                        let value = unsafe { std::ptr::read(&vals[k]) };
-                        let at = code.instructions[shape.stores[k].0 as usize].arg;
+                        let value = unsafe { std::ptr::read(v) };
+                        let at = code.instructions[store.0 as usize].arg;
                         if let Some(name) = code.names.get(at as usize) {
                             i.slot_set(name, value);
                         }
@@ -18246,14 +18255,17 @@ impl Interpreter {
                 && unsafe { i.dict.split_peek_mut() }.is_some_and(|split| {
                     let at = vals.as_ptr();
                     // SAFETY: as above.
-                    split.fill_fresh(keys, || ty.shared_keys.share(), n, |k| unsafe {
-                        at.add(k).read()
-                    })
+                    split.fill_fresh(
+                        keys,
+                        || ty.shared_keys.share(),
+                        n,
+                        |k| unsafe { at.add(k).read() },
+                    )
                 });
             if !filled {
-                for k in 0..n {
+                for (k, v) in vals[..n].iter().enumerate() {
                     // SAFETY: as above.
-                    let value = unsafe { std::ptr::read(&vals[k]) };
+                    let value = unsafe { std::ptr::read(v) };
                     // SAFETY: a store with nothing else running; the
                     // positions are the instance's next ones (proved above).
                     if let Err(value) = unsafe { i.split_append(k, value) } {
@@ -18317,8 +18329,7 @@ impl Interpreter {
         }
         // The arguments leave by plain decrements (whatever `__init__`
         // stored holds its own reference).
-        if !Self::core_all_droppable(&frame.stack[self_slot + 1..])
-        {
+        if !Self::core_all_droppable(&frame.stack[self_slot + 1..]) {
             return false;
         }
         let (inst, _) = self.alloc_plain_instance_obj(ty);
@@ -20697,8 +20708,7 @@ impl Interpreter {
                 }
             }
         }
-        if !Self::core_all_droppable(ops)
-        {
+        if !Self::core_all_droppable(ops) {
             return None;
         }
         #[cfg(test)]
@@ -20759,11 +20769,14 @@ impl Interpreter {
         if depth >= crate::recursion::recursion_limit() {
             return None;
         }
-        if !Self::core_all_droppable(ops)
-        {
+        if !Self::core_all_droppable(ops) {
             return None;
         }
-        let first = if matches!(ops[1], Object::Unbound) { 2 } else { 1 };
+        let first = if matches!(ops[1], Object::Unbound) {
+            2
+        } else {
+            1
+        };
         let nargs = ops.len() - first;
         let mut args: [*const Object; 8] = [std::ptr::null(); 8];
         for (k, o) in ops[first..].iter().enumerate() {
@@ -20820,8 +20833,7 @@ impl Interpreter {
         if unsafe { (*depth_cell).get() } >= crate::recursion::recursion_limit() {
             return None;
         }
-        if !Self::core_all_droppable(ops)
-        {
+        if !Self::core_all_droppable(ops) {
             return None;
         }
         // Its keyword-only parameters follow the positional ones, then its
@@ -21456,7 +21468,12 @@ impl Interpreter {
     /// `SEND` does (see [`Self::core_send_collapse`]), or resumes its
     /// parked one. Anything else is declined, touching nothing.
     #[inline(never)]
-    fn core_iter_collapse(&mut self, sw: &mut CoreSwitch, pc: usize, mode: InlineResume) -> SendHop {
+    fn core_iter_collapse(
+        &mut self,
+        sw: &mut CoreSwitch,
+        pc: usize,
+        mode: InlineResume,
+    ) -> SendHop {
         // SAFETY: see `CoreSwitch`: the running activation is synced, the
         // generator where `mode` puts it (checked by the caller).
         let frame = unsafe { &*sw.cur };
@@ -22213,7 +22230,7 @@ impl Interpreter {
         ) else {
             unreachable!("checked by `collapse_frame`");
         };
-        debug_assert!(std::ptr::eq::<Frame>(&*boxed, inner_frame));
+        debug_assert!(std::ptr::eq::<Frame>(&raw const *boxed, inner_frame));
         boxed.gen_first_resume = false;
         // (A step that stopped partway, or the innermost level's delegate's
         // return delivered by one, already gave it its value.)
@@ -22400,7 +22417,7 @@ impl Interpreter {
     fn collapse_park(&mut self, done: &mut InlineAct) {
         let mut rec = done.chain.take().expect("a collapsed chain");
         debug_assert!(rec.levels.iter().all(|l| l.act.shell.is_none()));
-        debug_assert!(self.lean_pending.len() == rec.pend_at + rec.levels.len());
+        debug_assert_eq!(self.lean_pending.len(), rec.pend_at + rec.levels.len());
         self.lean_pending.truncate(rec.pend_at);
         crate::recursion::leave_n(crate::recursion::depth_cell(), rec.levels.len());
         rec.parked = true;
@@ -22410,8 +22427,10 @@ impl Interpreter {
         drop(hold);
     }
 
-    /// An empty [`ChainRec`], from the pool or fresh.
+    /// An empty [`ChainRec`], from the pool or fresh (boxed: see
+    /// [`Self::chain_pool`]).
     #[inline]
+    #[allow(clippy::unnecessary_box_returns)]
     fn chain_rec(&mut self) -> Box<ChainRec> {
         self.chain_pool.pop().unwrap_or_else(|| {
             Box::new(ChainRec {
@@ -22476,9 +22495,10 @@ impl Interpreter {
             Rc::from_raw(level.gen)
         };
         // SAFETY: delegating (see `Delegate`); nothing borrows the state.
-        let GeneratorState::Delegating(boxed) =
-            std::mem::replace(unsafe { &mut *gen.state_raw().as_ptr() }, GeneratorState::Running)
-        else {
+        let GeneratorState::Delegating(boxed) = std::mem::replace(
+            unsafe { &mut *gen.state_raw().as_ptr() },
+            GeneratorState::Running,
+        ) else {
             unreachable!("a collapsed level is delegating");
         };
         let mut act = self.inline_slot();
@@ -23419,7 +23439,11 @@ impl Interpreter {
             && (crate::object::is_function_slot(n)
                 || matches!(
                     &**n,
-                    "__code__" | "__globals__" | "__closure__" | "__builtins__" | "__dict__"
+                    "__code__"
+                        | "__globals__"
+                        | "__closure__"
+                        | "__builtins__"
+                        | "__dict__"
                         | "__class__"
                 ))
         {
@@ -23608,8 +23632,7 @@ impl Interpreter {
                 return None;
             }
             let args = ops.get(2..2 + argc)?;
-            if !Self::core_all_droppable(ops)
-            {
+            if !Self::core_all_droppable(ops) {
                 return None;
             }
             return if cls.flags.is_builtin {
@@ -23627,8 +23650,7 @@ impl Interpreter {
         } else {
             (&[], &[])
         };
-        if !Self::core_all_droppable(ops)
-        {
+        if !Self::core_all_droppable(ops) {
             return None;
         }
         crate::stdlib::datetime_native::construct_names(cls, args, names, values)?.ok()
@@ -23640,8 +23662,7 @@ impl Interpreter {
     /// for the full handler, which also raises every error.
     #[inline(never)]
     fn core_native_method_kw(ops: &[Object], argc: usize) -> Option<Object> {
-        if !Self::core_all_droppable(ops)
-        {
+        if !Self::core_all_droppable(ops) {
             return None;
         }
         crate::stdlib::decimal_native::method_kw(ops, argc)
@@ -23691,9 +23712,7 @@ impl Interpreter {
             bound[i] = Some(value);
         }
         let n = bound.iter().take_while(|a| a.is_some()).count();
-        if bound[n..].iter().any(Option::is_some)
-            || !Self::core_all_droppable(ops)
-        {
+        if bound[n..].iter().any(Option::is_some) || !Self::core_all_droppable(ops) {
             return None;
         }
         let mut args = Vec::with_capacity(n + 1);
@@ -23718,8 +23737,7 @@ impl Interpreter {
             return None;
         }
         let fast = crate::stdlib::datetime_native::bound_fast(&bm.function, &bm.receiver)?;
-        if !Self::core_all_droppable(ops)
-        {
+        if !Self::core_all_droppable(ops) {
             return None;
         }
         // Bitwise views of the receiver and the arguments, never dropped:
@@ -28087,11 +28105,7 @@ impl Interpreter {
             Object::Module(m)
                 if !name.starts_with("__") && !crate::object::module_classes_possible() =>
             {
-                return m
-                    .dict
-                    .borrow()
-                    .get(&crate::object::StrKey(name))
-                    .cloned();
+                return m.dict.borrow().get(&crate::object::StrKey(name)).cloned();
             }
             // A built-in kind's method (`hasattr(stream, "write")`): its
             // lookup runs no code, and a found method means the attribute
@@ -28581,14 +28595,7 @@ impl Interpreter {
                         let v = Self::clone_operand(v);
                         drop(slots);
                         if let Some(ext) = code_vm_ext_existing(code) {
-                            slot_field_note(
-                                code,
-                                ext,
-                                cache_pc as usize,
-                                inst,
-                                key_idx,
-                                name_idx,
-                            );
+                            slot_field_note(code, ext, cache_pc as usize, inst, key_idx, name_idx);
                         }
                         Some(v)
                     }
@@ -45904,7 +45911,7 @@ impl Interpreter {
                 Ok(v)
             }
             Ok(FrameOutcome::Returned(v)) => {
-                Self::finish_running_gen(&gen);
+                Self::finish_running_gen(gen);
                 let frame: &mut Frame = &mut boxed;
                 self.recycle_frame_allocs(frame);
                 Self::release_finished_gen(gen);
@@ -49041,10 +49048,10 @@ impl Interpreter {
     ) -> Result<Object, RuntimeError> {
         specialize::record_miss(OpCode::LoadAttr as u8);
         // (An even count: see `specialized_load_attr`'s `Cooldown` arm.)
-        frame
-            .code
-            .caches
-            .set(cache_pc, weavepy_compiler::InlineCache::Cooldown(2 * COOLDOWN));
+        frame.code.caches.set(
+            cache_pc,
+            weavepy_compiler::InlineCache::Cooldown(2 * COOLDOWN),
+        );
         let name = Self::name_ref(&frame.code, name_idx)?;
         self.load_attr(&receiver, name)
     }
@@ -56923,8 +56930,7 @@ impl Interpreter {
                 // `SlotStorage::exception`), so a raise fills its slots in
                 // place. As there, only a non-atomic argument can close a
                 // cycle through the instance.
-                if matches!(&args, Object::Tuple(t) if t.iter().any(|x| !gc_trace::is_atomic(x)))
-                {
+                if matches!(&args, Object::Tuple(t) if t.iter().any(|x| !gc_trace::is_atomic(x))) {
                     i.ensure_gc_tracked();
                 }
                 *i.slots.borrow_mut() = crate::types::SlotStorage::exception(args, []);
@@ -75355,7 +75361,10 @@ pub(crate) fn load_method_builtin(
 
 /// The builtin type the `CALL` at `pc` last constructed in place.
 #[cfg(feature = "jit")]
-pub(crate) fn call_site_type(ext: &CodeConstObjects, pc: usize) -> Option<Rc<crate::types::TypeObject>> {
+pub(crate) fn call_site_type(
+    ext: &CodeConstObjects,
+    pc: usize,
+) -> Option<Rc<crate::types::TypeObject>> {
     let slot = ext.method_slot(pc)?;
     // SAFETY: GIL-serialized; the borrow ends here.
     match unsafe { &(*slot.0.get()).1 } {
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 3b7a6a55..218ed8ba 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -5766,7 +5766,9 @@ impl Drop for PyGenerator {
         let state = self.gen_state.get_mut();
         if matches!(
             state,
-            GeneratorState::Suspended(_) | GeneratorState::Created(_) | GeneratorState::Delegating(_)
+            GeneratorState::Suspended(_)
+                | GeneratorState::Created(_)
+                | GeneratorState::Delegating(_)
         ) {
             defer_generator_state_drop(std::mem::replace(state, GeneratorState::Finished));
         }
diff --git a/crates/weavepy-vm/src/stdlib/asyncio_events.rs b/crates/weavepy-vm/src/stdlib/asyncio_events.rs
index 3e38abfc..64c234ca 100644
--- a/crates/weavepy-vm/src/stdlib/asyncio_events.rs
+++ b/crates/weavepy-vm/src/stdlib/asyncio_events.rs
@@ -257,12 +257,10 @@ fn names() -> &'static Names {
 #[inline]
 fn inst_attr_with<R>(inst: &PyInstance, n: &AttrName, f: impl FnOnce(Option<&Object>) -> R) -> R {
     match inst.dict.published() {
-        Some(d) => f(d
-            .borrow()
-            .get(&crate::object::StrKeyHashed {
-                s: n.name,
-                hash: n.hash,
-            })),
+        Some(d) => f(d.borrow().get(&crate::object::StrKeyHashed {
+            s: n.name,
+            hash: n.hash,
+        })),
         None => {
             // SAFETY: a read that runs no code; the view ends with `f`,
             // which only inspects or clones the value.
@@ -570,10 +568,7 @@ fn loop_create_task_method(
             let Object::Instance(li) = loop_ else {
                 unreachable!("plain_loop checks for an instance")
             };
-            let no_factory = matches!(
-                inst_attr(li, &names().task_factory),
-                Some(Object::None)
-            );
+            let no_factory = matches!(inst_attr(li, &names().task_factory), Some(Object::None));
             let native =
                 global_module_type_ptr(&ls.globals, &names().tasks, &names().task, &TASK_MEMO)
                     == crate::stdlib::asyncio_mod::task_class_addr();
diff --git a/crates/weavepy-vm/src/stdlib/faulthandler_mod.rs b/crates/weavepy-vm/src/stdlib/faulthandler_mod.rs
index d773b85d..b66808b1 100644
--- a/crates/weavepy-vm/src/stdlib/faulthandler_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/faulthandler_mod.rs
@@ -97,11 +97,7 @@ pub fn note_thread_start(
 /// `sys._current_exceptions()`. Safe to materialize under the GIL:
 /// peer threads are parked, exactly as during a faulthandler dump.
 #[allow(clippy::type_complexity)]
-pub fn thread_snapshots() -> Vec<(
-    u64,
-    crate::object::FrameStack,
-    Rc<RefCell<Vec<Object>>>,
-)> {
+pub fn thread_snapshots() -> Vec<(u64, crate::object::FrameStack, Rc<RefCell<Vec<Object>>>)> {
     THREADS
         .lock()
         .unwrap()
diff --git a/crates/weavepy-vm/src/stdlib/functools_mod.rs b/crates/weavepy-vm/src/stdlib/functools_mod.rs
index de3ae590..3361449f 100644
--- a/crates/weavepy-vm/src/stdlib/functools_mod.rs
+++ b/crates/weavepy-vm/src/stdlib/functools_mod.rs
@@ -502,10 +502,12 @@ impl LruField {
         static KEYS: std::sync::OnceLock<[crate::shared_value::SharedStr; 7]> =
             std::sync::OnceLock::new();
         &KEYS.get_or_init(|| {
-            LruField::ALL.map(|field| match crate::stdlib::sys::intern_name(field.name()) {
-                Object::Str(s) => s,
-                _ => unreachable!("interned names are strs"),
-            })
+            LruField::ALL.map(
+                |field| match crate::stdlib::sys::intern_name(field.name()) {
+                    Object::Str(s) => s,
+                    _ => unreachable!("interned names are strs"),
+                },
+            )
         })[self as usize]
     }
 
@@ -793,10 +795,7 @@ fn scalar_lru_links(
 ) -> Result<Option<Rc<RefCell<Vec<u8>>>>, RuntimeError> {
     let mut dict = inst.dict_cell().borrow_mut();
     if lru_slot(&dict, LruField::State).is_none() {
-        dict.insert(
-            LruField::State.key_object(),
-            Object::Bool(false),
-        );
+        dict.insert(LruField::State.key_object(), Object::Bool(false));
     }
     let Some(state) = lru_slot_mut(&mut dict, LruField::State) else {
         unreachable!("just inserted")
@@ -904,9 +903,8 @@ fn scalar_lru_operation(
             None => {
                 // Deferral also protects against private-cache tampering: a
                 // foreign key must not invoke Python inside the paired borrow.
-                let (found, deferred) = crate::object::key_cmp_deferred(|| {
-                    data.get_index_of(&DictKey(key.clone()))
-                });
+                let (found, deferred) =
+                    crate::object::key_cmp_deferred(|| data.get_index_of(&DictKey(key.clone())));
                 let found = found?;
                 if deferred {
                     return Err(invalid_lru_order());
@@ -1004,7 +1002,9 @@ fn lru_fast_lookup(
         None if probe.miss_is_exact() => {
             let mut dict = inst.dict_cell().borrow_mut();
             lru_bump(&mut dict, LruField::Misses);
-            Some(FastLookup::Miss(lru_slot(&dict, LruField::Wrapped).cloned()))
+            Some(FastLookup::Miss(
+                lru_slot(&dict, LruField::Wrapped).cloned(),
+            ))
         }
         None => None,
     }
diff --git a/crates/weavepy-vm/src/stdlib/functools_mod/lru_order.rs b/crates/weavepy-vm/src/stdlib/functools_mod/lru_order.rs
index 9999cdda..59fbc038 100644
--- a/crates/weavepy-vm/src/stdlib/functools_mod/lru_order.rs
+++ b/crates/weavepy-vm/src/stdlib/functools_mod/lru_order.rs
@@ -48,8 +48,15 @@ impl<'a> Order<'a> {
             return None;
         }
         // SAFETY: the buffer holds `words` whole words (checked by `new`,
-        // kept by `push` and `swap_remove`).
-        Some(unsafe { self.bytes.as_ptr().cast::<usize>().add(index).read_unaligned() })
+        // kept by `push` and `swap_remove`); the read is unaligned.
+        #[allow(clippy::cast_ptr_alignment)]
+        Some(unsafe {
+            self.bytes
+                .as_ptr()
+                .cast::<usize>()
+                .add(index)
+                .read_unaligned()
+        })
     }
 
     #[inline(always)]
@@ -57,7 +64,8 @@ impl<'a> Order<'a> {
         if index >= self.words {
             return None;
         }
-        // SAFETY: as in `word`.
+        // SAFETY: as in `word`; the write is unaligned.
+        #[allow(clippy::cast_ptr_alignment)]
         unsafe {
             self.bytes
                 .as_mut_ptr()
diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs
index ba7777c9..8ba876f9 100644
--- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs
+++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs
@@ -14,7 +14,6 @@
 mod classes;
 mod encode;
 
-
 use crate::shared_value::{SharedSlice, SharedStr};
 use num_bigint::BigInt;
 use weavepy_compiler::CodeObject;
diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel/classes.rs b/crates/weavepy-vm/src/stdlib/pickle_accel/classes.rs
index 89784542..4dbed92b 100644
--- a/crates/weavepy-vm/src/stdlib/pickle_accel/classes.rs
+++ b/crates/weavepy-vm/src/stdlib/pickle_accel/classes.rs
@@ -5,7 +5,6 @@
 //! could require a Python callback reports `None` so that the caller returns
 //! to the full pickler or unpickler.
 
-
 use crate::builtin_types::builtin_types;
 use crate::object::{py_str_hash, DictData, Object, StrKey};
 use crate::sync::{Rc, RefCell};
diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs
index d0dd0c5d..0bda778a 100644
--- a/crates/weavepy-vm/src/stdlib/sys.rs
+++ b/crates/weavepy-vm/src/stdlib/sys.rs
@@ -400,11 +400,7 @@ pub fn build_with_state(
                         {
                             // 3.12+: the value is the handled exception
                             // *instance* (None outside any `except`).
-                            let value = exc
-                                .borrow()
-                                .last()
-                                .cloned()
-                                .unwrap_or(Object::None);
+                            let value = exc.borrow().last().cloned().unwrap_or(Object::None);
                             d.insert(DictKey(Object::Int(ident as i64)), value);
                         }
                         Ok(Object::Dict(Rc::new(RefCell::new(d))))
@@ -1660,9 +1656,7 @@ fn sys_getframemodulename(
 /// `sys.exception()` (PEP 3134 / 3.11+): the exception instance currently
 /// being handled, or `None` if not in an `except`. Equivalent to
 /// `sys.exc_info()[1]`. The verbatim CPython `contextlib` relies on this.
-fn sys_exception(
-    exc_info_stack: &Rc<RefCell<Vec<Object>>>,
-) -> Result<Object, RuntimeError> {
+fn sys_exception(exc_info_stack: &Rc<RefCell<Vec<Object>>>) -> Result<Object, RuntimeError> {
     let stack = exc_info_stack.borrow();
     Ok(stack
         .last()
@@ -1681,9 +1675,7 @@ fn capi_handled_exception_instance() -> Option<Object> {
     crate::foreign::capi_handled_exception()
 }
 
-fn sys_exc_info(
-    exc_info_stack: &Rc<RefCell<Vec<Object>>>,
-) -> Result<Object, RuntimeError> {
+fn sys_exc_info(exc_info_stack: &Rc<RefCell<Vec<Object>>>) -> Result<Object, RuntimeError> {
     let stack = exc_info_stack.borrow();
     let inst = stack
         .last()
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index b2f13658..eaf41e3b 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -6330,9 +6330,7 @@ unsafe fn try_native_call(
             if finalizer_queued() {
                 ctx.parked = Some(match expect {
                     SlotTag::Int | SlotTag::Float | SlotTag::Bool => unpack(bits, expect_tag),
-                    SlotTag::ObjPin if bits != u64::MAX => {
-                        unpack_pins(bits, expect_tag, &ctx.pins)
-                    }
+                    SlotTag::ObjPin if bits != u64::MAX => unpack_pins(bits, expect_tag, &ctx.pins),
                     _ => Object::None,
                 });
                 return Some(CallStatus::Boxed as i64);
diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs
index d64c2569..390614d3 100644
--- a/crates/weavepy-vm/src/types.rs
+++ b/crates/weavepy-vm/src/types.rs
@@ -1530,17 +1530,19 @@ impl TypeObject {
 
     /// [`Self::slot_layout`], decided now if it isn't yet.
     #[inline]
-    fn slot_layout_init(&self) -> &Option<SharedSlice<DictKey>> {
-        self.slot_layout.get_or_init(|| {
-            let layout = self.member_slot_layout();
-            if let Some(layout) = &layout {
-                self.slot_layout_word.store(
-                    SharedSlice::word(layout),
-                    std::sync::atomic::Ordering::Relaxed,
-                );
-            }
-            layout
-        })
+    fn slot_layout_init(&self) -> Option<&SharedSlice<DictKey>> {
+        self.slot_layout
+            .get_or_init(|| {
+                let layout = self.member_slot_layout();
+                if let Some(layout) = &layout {
+                    self.slot_layout_word.store(
+                        SharedSlice::word(layout),
+                        std::sync::atomic::Ordering::Relaxed,
+                    );
+                }
+                layout
+            })
+            .as_ref()
     }
 
     /// Whether the class is one whose native method bodies read its slots
@@ -2900,7 +2902,11 @@ impl SlotStorage {
     /// that size (the caller builds fresh storage).
     #[inline]
     pub(crate) fn reuse_unset_over(&mut self, layout: &SharedSlice<DictKey>) -> bool {
-        let SlotData::Fixed { layout: own, values } = &mut self.data else {
+        let SlotData::Fixed {
+            layout: own,
+            values,
+        } = &mut self.data
+        else {
             return false;
         };
         if values.len() != layout.len() {
@@ -4264,9 +4270,7 @@ impl PyInstance {
         if let Some(place) = slots.exc_tail_mut(which) {
             return std::mem::replace(place, value);
         }
-        slots
-            .insert(which.name(), value)
-            .unwrap_or(Object::Unbound)
+        slots.insert(which.name(), value).unwrap_or(Object::Unbound)
     }
 
     /// Delete slot `name` from the side table; `false` when unset.

From 59674d98285b1aa310b641fc0307f8e27e80daac Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Mon, 19 Oct 2026 08:40:36 -0700
Subject: [PATCH 165/173] perf: run calling effect leaves in place from the
 core loop

Since "call effect leaves that call directly", core_local_method declined
an effect leaf (a method body that stores) whose body itself calls, so
the call took a direct activation instead of the frameless leaf
evaluator. That pays off for compiled callers, whose direct call is
cheap (go -6.9%), but the core loop's alternative is a full activation:
the interpreter ran deltablue about 10% slower than before the change.

The decline is now a const parameter: compiled code's h_local_method
keeps it, and the core loop runs those leaves in place again.

deltablue (bench fixture, work 50, both JITs off), instructions retired:
2036M before this PR's change, 1924M at the previous head, 1754M now;
main is 1826M. The pybench suite is unchanged within noise.
---
 crates/weavepy-vm/src/frame_jit.rs |  2 +-
 crates/weavepy-vm/src/lib.rs       | 25 +++++++++++++++++--------
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs
index 7ee92a21..0152e2b7 100644
--- a/crates/weavepy-vm/src/frame_jit.rs
+++ b/crates/weavepy-vm/src/frame_jit.rs
@@ -3180,7 +3180,7 @@ unsafe extern "C" fn h_local_method(
         let st = &mut *st;
         let (code, ext, pc) = (&*code, &*ext, pc as usize);
         let local = &*st.locals.add(code.instructions[pc].arg as usize);
-        let Some((r, call_pc)) = (*st.interp).core_local_method(
+        let Some((r, call_pc)) = (*st.interp).core_local_method::<false>(
             code,
             Some(ext),
             local,
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 724caaa8..5049d6a5 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -11380,13 +11380,19 @@ impl Interpreter {
     /// `CALL`'s pc; `None` touches nothing (the instructions run one by
     /// one).
     ///
+    /// `CALLING_EFFECTS` runs an effect leaf (one that stores) whose body
+    /// itself calls in place too. Compiled callers pass `false`: for them
+    /// the method load and direct call cost less than the leaf evaluator
+    /// with its callee nested in it. The core loop passes `true`, since
+    /// its alternative is a full activation, which costs more.
+    ///
     /// # Safety
     ///
     /// `lbase` is the running activation's `nlocals` locals; nothing else
     /// borrows them.
     #[inline(always)]
     #[allow(clippy::too_many_arguments)]
-    unsafe fn core_local_method(
+    unsafe fn core_local_method<const CALLING_EFFECTS: bool>(
         &self,
         code: &CodeObject,
         ext: Option<&CodeConstObjects>,
@@ -11401,15 +11407,16 @@ impl Interpreter {
         let mslots: Sites<'_, MethodSlot> = ext.map_or(Sites::empty(), |e| e.method_sites());
         match other {
             Object::Instance(_) | Object::Module(_) => {
-                // A site that verified its pure callee for this class version
-                // (its call checks the argument shape itself). An effect
-                // leaf (one that stores) costs more evaluated frameless than
-                // called: the method loads and its call runs directly.
+                // A site that verified its callee for this class version
+                // (its call checks the argument shape itself); a calling
+                // effect leaf only when `CALLING_EFFECTS` (see above).
                 let mut missed = false;
                 if let (Object::Instance(i), Some(ext)) = (other, ext) {
                     if let Some(site) = leaf_site_hit(ext, pc + 1, i.cls_raw().attr_version.get())
                         // SAFETY: the site's code is its live function's.
-                        .filter(|&(_, c, effect)| !effect || !code_calls(unsafe { &*c }))
+                        .filter(|&(_, c, effect)| {
+                            CALLING_EFFECTS || !effect || !code_calls(unsafe { &*c })
+                        })
                     {
                         match self.core_leaf_site_call(
                             code,
@@ -11465,7 +11472,9 @@ impl Interpreter {
                             .is_some_and(|fp| {
                                 let f = unsafe { &*fp };
                                 fn_is_pure_leaf(f)
-                                    || (fn_is_leaf(f) && !code_calls(unsafe { &*f.code.as_ptr() }))
+                                    || (fn_is_leaf(f)
+                                        && (CALLING_EFFECTS
+                                            || !code_calls(unsafe { &*f.code.as_ptr() })))
                             }),
                         _ => false,
                     };
@@ -11882,7 +11891,7 @@ impl Interpreter {
                                 // `x.m(...)` of a leaf or native method on the
                                 // borrowed local (see `core_local_method`).
                                 other if kind == FAST_METHOD => {
-                                    if let Some((r, call_pc)) = self.core_local_method(
+                                    if let Some((r, call_pc)) = self.core_local_method::<true>(
                                         code,
                                         ext,
                                         other,

From 27ad3f21ce997e1727ef2d58b453adea1f9e5d9c Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Mon, 19 Oct 2026 09:16:25 -0700
Subject: [PATCH 166/173] perf: send only recursive effect leaves to direct
 calls

Compiled code called an effect leaf (a method body that stores) directly
instead of through the frameless leaf evaluator whenever its body made
any call. That paid off for go's recursive Square.find, whose frameless
evaluation nests its own evaluation at every level. But an effect leaf
that calls other code is cheaper frameless, and as a real activation it
gathers call heat and compiles: deltablue's constraint methods compiled
seven more bodies in a short run, to no gain even in longer ones, and
the macOS bench gate measured deltablue 18% slower than main.

The test is now whether the body may call itself (it makes a call and
its names include its own); code_calls becomes code_recurses.

deltablue (bench fixture, work 50), instructions retired: 2136M -> 2044M
(main: 1977M). pybench suite geomean 0.9998, go unchanged.
---
 crates/weavepy-vm/src/lib.rs | 48 ++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 21 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 5049d6a5..5e037829 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -11380,11 +11380,12 @@ impl Interpreter {
     /// `CALL`'s pc; `None` touches nothing (the instructions run one by
     /// one).
     ///
-    /// `CALLING_EFFECTS` runs an effect leaf (one that stores) whose body
-    /// itself calls in place too. Compiled callers pass `false`: for them
-    /// the method load and direct call cost less than the leaf evaluator
-    /// with its callee nested in it. The core loop passes `true`, since
-    /// its alternative is a full activation, which costs more.
+    /// `CALLING_EFFECTS` runs a recursive effect leaf (one that stores and
+    /// may call itself, see [`code_recurses`]) in place too. Compiled
+    /// callers pass `false`: for them the method load and direct call cost
+    /// less than the leaf evaluator nesting its own evaluation. The core
+    /// loop passes `true`, since its alternative is a full activation,
+    /// which costs more.
     ///
     /// # Safety
     ///
@@ -11408,14 +11409,14 @@ impl Interpreter {
         match other {
             Object::Instance(_) | Object::Module(_) => {
                 // A site that verified its callee for this class version
-                // (its call checks the argument shape itself); a calling
+                // (its call checks the argument shape itself); a recursive
                 // effect leaf only when `CALLING_EFFECTS` (see above).
                 let mut missed = false;
                 if let (Object::Instance(i), Some(ext)) = (other, ext) {
                     if let Some(site) = leaf_site_hit(ext, pc + 1, i.cls_raw().attr_version.get())
                         // SAFETY: the site's code is its live function's.
                         .filter(|&(_, c, effect)| {
-                            CALLING_EFFECTS || !effect || !code_calls(unsafe { &*c })
+                            CALLING_EFFECTS || !effect || !code_recurses(unsafe { &*c })
                         })
                     {
                         match self.core_leaf_site_call(
@@ -11474,7 +11475,7 @@ impl Interpreter {
                                 fn_is_pure_leaf(f)
                                     || (fn_is_leaf(f)
                                         && (CALLING_EFFECTS
-                                            || !code_calls(unsafe { &*f.code.as_ptr() })))
+                                            || !code_recurses(unsafe { &*f.code.as_ptr() })))
                             }),
                         _ => false,
                     };
@@ -74523,8 +74524,8 @@ struct CodeConstObjects {
     /// The body's store-only `__init__` shape (see [`code_store_init`]),
     /// or `None` when it isn't one.
     store_init: std::sync::OnceLock<Option<Box<StoreInit>>>,
-    /// Whether the body makes a call (see [`code_calls`]): `0` not yet
-    /// decided, `1` no, `2` yes.
+    /// Whether the body may call itself (see [`code_recurses`]): `0` not
+    /// yet decided, `1` no, `2` yes.
     calls: std::sync::atomic::AtomicU8,
     /// Per `co_names` entry, its position among the shared names of the
     /// instances last checked for shadowing it (see [`inst_may_shadow`]).
@@ -75405,7 +75406,7 @@ pub(crate) fn call_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
 /// without an activation of its own.
 #[cfg(feature = "jit")]
 pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
-    // (An effect leaf that calls runs as an ordinary call, not in place:
+    // (A recursive effect leaf runs as an ordinary call, not in place:
     // `core_local_method` declines it, so a native method-call helper
     // there would only cost every call its probes.)
     // SAFETY: GIL-serialized; the borrows end before any refill; a live
@@ -75415,7 +75416,7 @@ pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
         .get(pc)
         .and_then(|s| unsafe { &*s.0.get() }.as_ref())
         .is_some_and(|site| {
-            site.func.strong_count() > 0 && !(site.effect && code_calls(unsafe { &*site.code }))
+            site.func.strong_count() > 0 && !(site.effect && code_recurses(unsafe { &*site.code }))
         });
     leaf || ext
         .method_slot(pc)
@@ -75425,7 +75426,7 @@ pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool {
                 // SAFETY: the function is alive while the upgrade holds it.
                 w.upgrade().is_some_and(|f| {
                     fn_is_pure_leaf(&f)
-                        || (fn_is_leaf(&f) && !code_calls(unsafe { &*f.code.as_ptr() }))
+                        || (fn_is_leaf(&f) && !code_recurses(unsafe { &*f.code.as_ptr() }))
                 })
             }
             _ => false,
@@ -76925,10 +76926,14 @@ fn code_returns_only_none(code: &CodeObject) -> bool {
     yes
 }
 
-/// Whether `code`'s body makes a call of any kind (cached in its
-/// extension). An effect leaf that calls costs more evaluated frameless,
-/// its callee's evaluation nested in it, than called directly.
-fn code_calls(code: &CodeObject) -> bool {
+/// Whether `code`'s body may call itself: it makes a call, and its names
+/// include its own (`self.find(...)` in a method `find`, or a recursive
+/// function's global). Cached in its extension. Evaluated frameless, a
+/// recursive effect leaf nests its own evaluation at every level, which
+/// costs more than calling it directly (go's `Square.find`); an effect
+/// leaf that calls other code costs less frameless (deltablue's
+/// constraint methods, whose direct calls compiled them to no gain).
+fn code_recurses(code: &CodeObject) -> bool {
     use std::sync::atomic::Ordering::Relaxed;
     let Some(ext) = code_vm_ext(code) else {
         return true;
@@ -76938,10 +76943,11 @@ fn code_calls(code: &CodeObject) -> bool {
         2 => return true,
         _ => {}
     }
-    let yes = code
-        .instructions
-        .iter()
-        .any(|i| matches!(i.op, OpCode::Call | OpCode::CallKw | OpCode::CallEx));
+    let yes = code.names.contains(&code.name)
+        && code
+            .instructions
+            .iter()
+            .any(|i| matches!(i.op, OpCode::Call | OpCode::CallKw | OpCode::CallEx));
     ext.calls.store(if yes { 2 } else { 1 }, Relaxed);
     yes
 }

From ba20d3253df63e4d9212571b9d9926999657b744 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Sun, 18 Oct 2026 21:51:54 -0700
Subject: [PATCH 167/173] perf: drop purged pages from the resident size on
 macOS

mimalloc purges freed pages on macOS with MADV_FREE_REUSABLE, which
lowers the footprint but leaves the pages in the resident size, and so
in ru_maxrss, until the system reclaims them: freed memory never left a
WeavePy process's RSS. A test mapping of 64 MB stayed at 66 MB resident
after MADV_FREE_REUSABLE, MADV_DONTNEED and MADV_FREE alike, and fell to
0.6 MB only when fresh pages were mapped over it. No runtime option does
that: purge_decommits, purge_delay, page_full_retain,
page_commit_on_demand and arena_reserve left chaos, go and float_points
where they were, and disallow_arena_alloc (a map and an unmap per page)
cost 0.55% more instructions over the suite.

WeavePy now vendors libmimalloc-sys 0.1.49, trimmed to the mimalloc 3
sources it compiles, with one patch in _mi_prim_decommit: on macOS a
purge maps fresh pages over the range, falling back to upstream's
advice if that fails. vendor/libmimalloc-sys/README.md records the
upstream version, the reason and the exact diff. On macOS the allocator
also sets mimalloc's purge delay to 10 ms (from 1000 ms, longer than a
short program's warm-up; MIMALLOC_PURGE_DELAY still overrides it).
Linux and Windows build the same C code paths and options as before.

Peak RSS against the previous commit (quick builds, warm caches): chaos
27.5 to 25.0 MB, go 26.8 to 24.4 MB, float_points 58.9 to 45.4 MB. Over
the suite this change cut the RSS geomean by about 7%. Instructions per
call: +0.2% geomean, from refaulting purged pages (float_points +3.1%,
async_tree +1.8%, graph_search +1.4%).

It should also fix Pillow's TestAvifLeaks::test_leak_load on the 3-core
macOS CI runners, which bounds peak RSS growth over one AVIF decode and
failed there (never on Linux, where purging already drops pages). With
freed memory resident, WeavePy sat at its peak, so the decoder's
system-allocator memory set a new one: after freeing a large batch of
Python objects, one decode raised the peak by 1.2 MB (CPython: 0), and
now by 0. The bench gate's fixtures run no slower against the previous
commit (list_ops, generators and float_math 7-9% faster).
---
 .gitignore                                    |    4 +-
 Cargo.lock                                    |    2 -
 Cargo.toml                                    |    5 +
 crates/weavepy-cli/src/alloc.rs               |   23 +
 vendor/libmimalloc-sys/Cargo.toml             |   79 +
 vendor/libmimalloc-sys/LICENSE.txt            |   25 +
 vendor/libmimalloc-sys/README.md              |   81 +
 vendor/libmimalloc-sys/build.rs               |  137 +
 .../libmimalloc-sys/c_src/mimalloc/v3/LICENSE |   21 +
 .../mimalloc/v3/include/mimalloc-new-delete.h |   66 +
 .../mimalloc/v3/include/mimalloc-override.h   |   68 +
 .../mimalloc/v3/include/mimalloc-stats.h      |  168 ++
 .../c_src/mimalloc/v3/include/mimalloc.h      |  706 +++++
 .../mimalloc/v3/include/mimalloc/atomic.h     |  562 ++++
 .../c_src/mimalloc/v3/include/mimalloc/bits.h |  340 +++
 .../mimalloc/v3/include/mimalloc/internal.h   | 1282 +++++++++
 .../c_src/mimalloc/v3/include/mimalloc/prim.h |  542 ++++
 .../mimalloc/v3/include/mimalloc/track.h      |  150 +
 .../mimalloc/v3/include/mimalloc/types.h      |  732 +++++
 .../c_src/mimalloc/v3/src/alloc-aligned.c     |  441 +++
 .../c_src/mimalloc/v3/src/alloc-override.c    |  396 +++
 .../c_src/mimalloc/v3/src/alloc-posix.c       |  202 ++
 .../c_src/mimalloc/v3/src/alloc.c             |  885 ++++++
 .../c_src/mimalloc/v3/src/arena-meta.c        |  179 ++
 .../c_src/mimalloc/v3/src/arena.c             | 2514 +++++++++++++++++
 .../c_src/mimalloc/v3/src/bitmap.c            | 1949 +++++++++++++
 .../c_src/mimalloc/v3/src/bitmap.h            |  343 +++
 .../c_src/mimalloc/v3/src/free.c              |  658 +++++
 .../c_src/mimalloc/v3/src/heap.c              |  273 ++
 .../c_src/mimalloc/v3/src/init.c              | 1199 ++++++++
 .../c_src/mimalloc/v3/src/libc.c              |  477 ++++
 .../c_src/mimalloc/v3/src/options.c           |  704 +++++
 .../c_src/mimalloc/v3/src/os.c                |  923 ++++++
 .../c_src/mimalloc/v3/src/page-map.c          |  447 +++
 .../c_src/mimalloc/v3/src/page-queue.c        |  458 +++
 .../c_src/mimalloc/v3/src/page.c              | 1039 +++++++
 .../mimalloc/v3/src/prim/emscripten/prim.c    |  261 ++
 .../v3/src/prim/osx/alloc-override-zone.c     |  481 ++++
 .../c_src/mimalloc/v3/src/prim/osx/prim.c     |    9 +
 .../c_src/mimalloc/v3/src/prim/prim.c         |   76 +
 .../c_src/mimalloc/v3/src/prim/readme.md      |    9 +
 .../c_src/mimalloc/v3/src/prim/unix/prim.c    | 1001 +++++++
 .../c_src/mimalloc/v3/src/prim/wasi/prim.c    |  291 ++
 .../v3/src/prim/windows/etw-mimalloc.wprp     |   61 +
 .../c_src/mimalloc/v3/src/prim/windows/etw.h  |  905 ++++++
 .../mimalloc/v3/src/prim/windows/etw.man      |  Bin 0 -> 3926 bytes
 .../c_src/mimalloc/v3/src/prim/windows/prim.c | 1124 ++++++++
 .../mimalloc/v3/src/prim/windows/readme.md    |   17 +
 .../c_src/mimalloc/v3/src/random.c            |  258 ++
 .../c_src/mimalloc/v3/src/static.c            |   43 +
 .../c_src/mimalloc/v3/src/stats.c             |  813 ++++++
 .../c_src/mimalloc/v3/src/theap.c             |  715 +++++
 .../c_src/mimalloc/v3/src/threadlocal.c       |  236 ++
 vendor/libmimalloc-sys/src/extended.rs        | 1126 ++++++++
 vendor/libmimalloc-sys/src/lib.rs             |  139 +
 55 files changed, 25642 insertions(+), 3 deletions(-)
 create mode 100644 vendor/libmimalloc-sys/Cargo.toml
 create mode 100644 vendor/libmimalloc-sys/LICENSE.txt
 create mode 100644 vendor/libmimalloc-sys/README.md
 create mode 100644 vendor/libmimalloc-sys/build.rs
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/LICENSE
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-new-delete.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-override.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-stats.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/atomic.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/bits.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/internal.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/prim.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/track.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/types.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-aligned.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-override.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-posix.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena-meta.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/free.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/heap.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/init.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/libc.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/options.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/os.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-map.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-queue.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/emscripten/prim.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/alloc-override-zone.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/prim.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/prim.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/readme.md
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/wasi/prim.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw-mimalloc.wprp
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.h
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.man
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/prim.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/readme.md
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/random.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/static.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/stats.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/theap.c
 create mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/threadlocal.c
 create mode 100644 vendor/libmimalloc-sys/src/extended.rs
 create mode 100644 vendor/libmimalloc-sys/src/lib.rs

diff --git a/.gitignore b/.gitignore
index 7735b866..48c1aefb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -68,10 +68,12 @@ node_modules/
 # Ignore everything under vendor/ EXCEPT the lightly-patched lzma-sys crate,
 # which is a committed build input wired via Cargo.toml [patch.crates-io],
 # and the expat-sys crate (RFC 0056 WS3: vendored expat 2.6.4 + FFI bindings)
-# that backs the native `pyexpat` module.
+# that backs the native `pyexpat` module, and the patched libmimalloc-sys
+# crate (see Cargo.toml [patch.crates-io]).
 /vendor/*
 !/vendor/lzma-sys/
 !/vendor/expat-sys/
+!/vendor/libmimalloc-sys/
 
 # Instrumented build for scripts/gen-order-file.sh
 target-order/
diff --git a/Cargo.lock b/Cargo.lock
index a3fef29c..a12d4cbf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1265,8 +1265,6 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 [[package]]
 name = "libmimalloc-sys"
 version = "0.1.49"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
 dependencies = [
  "cc",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index b1a7f413..c18619c2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -209,8 +209,13 @@ proptest    = "1.5"
 # patch only flips on the already-compiled coders; the crate is otherwise byte-
 # for-byte lzma-sys 0.1.20 (heavy non-compiled trees like docs/po/tests/cli are
 # trimmed). The override applies transitively, so `xz2` picks it up too.
+#
+# It also vendors `libmimalloc-sys` 0.1.49 with a one-function macOS patch:
+# a purge maps fresh pages over the freed range, so freed memory leaves the
+# resident size (see `vendor/libmimalloc-sys/README.md` for why and the diff).
 [patch.crates-io]
 lzma-sys = { path = "vendor/lzma-sys" }
+libmimalloc-sys = { path = "vendor/libmimalloc-sys" }
 
 [workspace.lints.rust]
 unsafe_op_in_unsafe_fn = "warn"
diff --git a/crates/weavepy-cli/src/alloc.rs b/crates/weavepy-cli/src/alloc.rs
index a3a9dc22..1fafd60f 100644
--- a/crates/weavepy-cli/src/alloc.rs
+++ b/crates/weavepy-cli/src/alloc.rs
@@ -84,9 +84,28 @@ extern "C" {
     fn mi_option_set(option: c_int, value: c_long);
 }
 
+/// mimalloc 3's `mi_option_purge_delay` (its position in `mi_option_t`).
+#[cfg(target_os = "macos")]
+const MI_OPTION_PURGE_DELAY: c_int = 15;
+
 /// mimalloc 3's `mi_option_arena_reserve` (its position in `mi_option_t`).
 const MI_OPTION_ARENA_RESERVE: c_int = 23;
 
+/// How long freed pages wait before mimalloc purges them on macOS, in
+/// milliseconds, unless the environment sets `MIMALLOC_PURGE_DELAY`.
+///
+/// A purge there maps fresh pages over the range (the vendored crate's
+/// patch, see `vendor/libmimalloc-sys/README.md`), so it is what takes
+/// freed memory out of the resident size. (Elsewhere mimalloc's own
+/// default stands.)
+/// mimalloc's default waits 1000 ms, longer than a short program's whole
+/// warm-up: the transient memory of each JIT compile, and of the program's
+/// own bursts, stayed resident under every later peak (chaos peaked at
+/// 29 MB against 25 MB). The short wait still spares the steady churn of a
+/// loop that frees and reallocates the same pages from repeated faults.
+#[cfg(target_os = "macos")]
+const PURGE_DELAY_MS: c_long = 10;
+
 /// The size of each arena mimalloc reserves, in KiB, unless the
 /// environment sets `MIMALLOC_ARENA_RESERVE`.
 ///
@@ -110,5 +129,9 @@ pub extern "C" fn configure() {
         if libc::getenv(c"MIMALLOC_ARENA_RESERVE".as_ptr()).is_null() {
             mi_option_set(MI_OPTION_ARENA_RESERVE, ARENA_RESERVE_KIB);
         }
+        #[cfg(target_os = "macos")]
+        if libc::getenv(c"MIMALLOC_PURGE_DELAY".as_ptr()).is_null() {
+            mi_option_set(MI_OPTION_PURGE_DELAY, PURGE_DELAY_MS);
+        }
     }
 }
diff --git a/vendor/libmimalloc-sys/Cargo.toml b/vendor/libmimalloc-sys/Cargo.toml
new file mode 100644
index 00000000..db1d6f65
--- /dev/null
+++ b/vendor/libmimalloc-sys/Cargo.toml
@@ -0,0 +1,79 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g., crates.io) dependencies.
+#
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
+
+[package]
+edition = "2018"
+name = "libmimalloc-sys"
+version = "0.1.49"
+authors = ["Octavian Oncescu <octavonce@gmail.com>"]
+build = "build.rs"
+links = "mimalloc"
+exclude = [
+    "/c_src/mimalloc/v2/bin",
+    "/c_src/mimalloc/v2/cmake",
+    "/c_src/mimalloc/v2/doc",
+    "/c_src/mimalloc/v2/docs",
+    "/c_src/mimalloc/v2/ide",
+    "/c_src/mimalloc/v2/test",
+    "/c_src/mimalloc/v3/bin",
+    "/c_src/mimalloc/v3/cmake",
+    "/c_src/mimalloc/v3/doc",
+    "/c_src/mimalloc/v3/docs",
+    "/c_src/mimalloc/v3/ide",
+    "/c_src/mimalloc/v3/test",
+]
+autolib = false
+autobins = false
+autoexamples = false
+autotests = false
+autobenches = false
+description = "Sys crate wrapping the mimalloc allocator"
+readme = false
+keywords = [
+    "allocator",
+    "encrypted-heap",
+    "performance",
+]
+categories = [
+    "memory-management",
+    "api-bindings",
+]
+license = "MIT"
+repository = "https://github.com/purpleprotocol/mimalloc_rust/tree/master/libmimalloc-sys"
+
+[package.metadata.docs.rs]
+features = ["extended"]
+
+[features]
+arena = []
+debug = []
+debug_in_debug = []
+extended = ["cty"]
+local_dynamic_tls = []
+no_thp = []
+override = []
+secure = []
+v2 = []
+win_direct_tls = []
+
+[lib]
+name = "libmimalloc_sys"
+path = "src/lib.rs"
+
+[dependencies.cty]
+version = "0.2"
+optional = true
+
+[dev-dependencies.libc]
+version = "0.2"
+
+[build-dependencies.cc]
+version = "1.2"
diff --git a/vendor/libmimalloc-sys/LICENSE.txt b/vendor/libmimalloc-sys/LICENSE.txt
new file mode 100644
index 00000000..9a3377d9
--- /dev/null
+++ b/vendor/libmimalloc-sys/LICENSE.txt
@@ -0,0 +1,25 @@
+Copyright 2019 Octavian Oncescu
+
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/vendor/libmimalloc-sys/README.md b/vendor/libmimalloc-sys/README.md
new file mode 100644
index 00000000..6483af31
--- /dev/null
+++ b/vendor/libmimalloc-sys/README.md
@@ -0,0 +1,81 @@
+# Vendored `libmimalloc-sys`
+
+This is `libmimalloc-sys` 0.1.49 from crates.io (bundling mimalloc 3.3.2,
+`MI_MALLOC_VERSION 30302`) with one patch. The workspace's
+`[patch.crates-io]` table points at it, so the `weavepy` binary's allocator
+(`crates/weavepy-cli/src/alloc.rs`) builds from here on every platform.
+
+## Why it's vendored
+
+mimalloc returns freed memory to the system by purging it. On macOS,
+upstream purges with `madvise(MADV_FREE_REUSABLE)`, which lowers the
+process's footprint, but the pages stay in its resident size (and so in the
+peak `ru_maxrss`) until the system reclaims them under memory pressure.
+`MADV_DONTNEED` and `MADV_FREE` behave the same way. Only unmapping the
+range, or mapping fresh pages over it, takes them out. With 64 MB of touched
+pages, the resident size stayed at 66 MB after each of those advices and
+fell to 0.6 MB once fresh pages were mapped over the range.
+
+With the upstream purge, nothing freed ever left a WeavePy process's RSS,
+and no runtime option helps. `purge_decommits`, `purge_delay`,
+`page_full_retain`, `page_commit_on_demand` and `arena_reserve` made no
+difference. `disallow_arena_alloc` gets the RSS back, but it maps and
+unmaps every page: 0.55% more instructions over the benchmark suite. The
+patch, with a 10 ms purge delay set in `alloc.rs` (macOS only), brought the
+suite's peak RSS down about 7% for 0.22% more instructions.
+
+On Linux, mimalloc already purges with `MADV_DONTNEED`, which drops resident
+pages there, and Windows decommits. Neither path changes.
+
+## What differs from upstream
+
+- `c_src/mimalloc/v3/src/prim/unix/prim.c`: the patch below.
+- Removed, as nothing here compiles them: mimalloc 2's sources
+  (`c_src/mimalloc/v2`, so the crate's `v2` feature no longer builds) and,
+  under `c_src/mimalloc/v3`, everything but `include/`, `src/` and
+  `LICENSE`. Also removed: the published crate's `Cargo.toml.orig`,
+  `Cargo.lock`, `.cargo_vcs_info.json` and `.gitignore`.
+
+Everything else (`Cargo.toml`, `build.rs`, `src/`, `LICENSE.txt` and the
+compiled C sources) is byte-for-byte upstream.
+
+```diff
+--- libmimalloc-sys-0.1.49/c_src/mimalloc/v3/src/prim/unix/prim.c
++++ vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c
+@@ -492,10 +492,16 @@
+ 
+ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
+   int err = 0;
+   #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
+-    // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)
+-    err = unix_madvise(start, size, MADV_FREE_REUSABLE);
+-    if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
++    // decommit on macOS: map fresh zero pages over the range (WeavePy patch).
++    // MADV_FREE_REUSABLE only lowers the footprint: the pages stay in the
++    // resident size (and `ru_maxrss`) until the system reclaims them. If the
++    // mapping fails, the range is left as it was and marked reusable.
++    void* p = mmap(start, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, unix_mmap_fd(), 0);
++    if (p != start) {
++      err = unix_madvise(start, size, MADV_FREE_REUSABLE);
++      if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
++    }
+   #else
+     // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
+     err = unix_madvise(start, size, MADV_DONTNEED);
+```
+
+The mapping keeps the range readable and writable, as upstream's purge
+does, so `needs_recommit` stays false in release builds. A purged range's
+contents are undefined to mimalloc either way, so zero pages are a valid
+result. The mapping uses mimalloc's own VM tag (`unix_mmap_fd()`).
+
+## Updating
+
+To move to a newer `libmimalloc-sys`:
+
+1. Copy the published crate over this directory.
+2. Remove the files listed above.
+3. Reapply the patch to `_mi_prim_decommit`, and update this file.
+
+Cargo doesn't rebuild the C sources when only they change, so touch
+`build.rs` after editing them.
diff --git a/vendor/libmimalloc-sys/build.rs b/vendor/libmimalloc-sys/build.rs
new file mode 100644
index 00000000..0f3fcda9
--- /dev/null
+++ b/vendor/libmimalloc-sys/build.rs
@@ -0,0 +1,137 @@
+use std::env;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+fn main() {
+    let mut build = cc::Build::new();
+
+    let version = if env::var("CARGO_FEATURE_V2").is_ok() {
+        "v2"
+    } else {
+        "v3"
+    };
+
+    let cargo_manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set");
+    let include_root = Path::new(&cargo_manifest_dir)
+        .join("c_src")
+        .join("mimalloc")
+        .join(version);
+    let include_dir = include_root
+        .join("include")
+        .to_str()
+        .expect("include path is not valid UTF-8")
+        .to_string();
+    let static_source = include_root.join("src").join("static.c");
+
+    // Make the include directory available to consumers via the `DEP_MIMALLOC_INCLUDE_DIR`
+    // environment variable.
+    println!("cargo:INCLUDE_DIR={include_dir}");
+
+    build.include(format!("c_src/mimalloc/{version}/include"));
+    build.include(format!("c_src/mimalloc/{version}/src"));
+
+    let target_os = env::var("CARGO_CFG_TARGET_OS").expect("target_os not defined!");
+    let target_family = env::var("CARGO_CFG_TARGET_FAMILY").expect("target_family not defined!");
+    let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default();
+    let target_vendor = env::var("CARGO_CFG_TARGET_VENDOR").expect("target_vendor not defined!");
+    let target_arch = env::var("CARGO_CFG_TARGET_ARCH").expect("target_arch not defined!");
+    let cargo_debug = env::var("DEBUG")
+        .map(|value| value == "true" || value == "1")
+        .unwrap_or(false);
+    let debug_enabled = env::var_os("CARGO_FEATURE_DEBUG").is_some()
+        || (env::var_os("CARGO_FEATURE_DEBUG_IN_DEBUG").is_some() && cargo_debug);
+
+    if target_family != "windows" {
+        build.flag("-Wno-error=date-time");
+    }
+
+    if target_env == "msvc" {
+        // Mimalloc expects the MSVC/clang-cl build to use the C++ atomics path.
+        build.cpp(true);
+        build.std("c++17");
+        build.flag_if_supported("/Zc:__cplusplus");
+
+        let wrapper =
+            PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")).join("mimalloc-static.cc");
+        let include = static_source.to_string_lossy().replace('\\', "/");
+        fs::write(&wrapper, format!("#include \"{include}\"\n"))
+            .expect("failed to write mimalloc C++ wrapper");
+        build.file(wrapper);
+    } else {
+        build.file(&static_source);
+    }
+
+    let compiler = build.get_compiler();
+
+    if env::var_os("CARGO_FEATURE_OVERRIDE").is_some() {
+        // Overriding malloc is only available on windows in shared mode, but we
+        // only ever build a static lib.
+        if target_family != "windows" {
+            build.define("MI_MALLOC_OVERRIDE", None);
+        }
+        if target_vendor == "apple" {
+            build.define("MI_OSX_ZONE", Some("1"));
+            build.define("MI_OSX_INTERPOSE", Some("1"));
+        }
+        if !compiler.is_like_msvc() {
+            build.flag_if_supported("-fno-builtin-malloc");
+        }
+    }
+
+    if env::var_os("CARGO_FEATURE_SECURE").is_some() {
+        build.define("MI_SECURE", "4");
+    }
+
+    if target_os == "windows" && env::var_os("CARGO_FEATURE_WIN_DIRECT_TLS").is_some() {
+        build.define("MI_WIN_DIRECT_TLS", "1");
+    }
+
+    let dynamic_tls = env::var("CARGO_FEATURE_LOCAL_DYNAMIC_TLS").is_ok();
+
+    if target_family == "unix" && target_os != "haiku" {
+        if dynamic_tls {
+            build.flag_if_supported("-ftls-model=local-dynamic");
+        } else {
+            build.flag_if_supported("-ftls-model=initial-exec");
+        }
+    }
+
+    if (target_os == "linux" || target_os == "android")
+        && env::var_os("CARGO_FEATURE_NO_THP").is_some()
+    {
+        build.define("MI_NO_THP", "1");
+    }
+
+    if debug_enabled {
+        build.define("MI_DEBUG", "3");
+        build.define("MI_SHOW_ERRORS", "1");
+    } else {
+        // Remove heavy debug assertions etc.
+        build.define("MI_DEBUG", "0");
+        if !cargo_debug {
+            build.define("MI_BUILD_RELEASE", None);
+            build.define("NDEBUG", None);
+        }
+    }
+
+    build.compile("mimalloc");
+
+    // on armv6 we need to link with libatomic
+    if target_os == "linux" && target_arch == "arm" {
+        // Embrace the atomic capability library across various platforms.
+        // For instance, on certain platforms, llvm has relocated the atomic of the arm32 architecture to libclang_rt.builtins.a
+        // while some use libatomic.a, and others use libatomic_ops.a.
+        let atomic_name = env::var("DEP_ATOMIC").unwrap_or("atomic".to_owned());
+        println!("cargo:rustc-link-lib={}", atomic_name);
+    }
+
+    // Link with libs needed on Windows
+    if target_os == "windows" {
+        // https://github.com/microsoft/mimalloc/blob/af21001f7a65eafb8fb16460b018ebf9d75e2ad8/CMakeLists.txt#L487
+        let libs = ["psapi", "shell32", "user32", "advapi32", "bcrypt"];
+
+        for lib in libs {
+            println!("cargo:rustc-link-lib={}", lib);
+        }
+    }
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/LICENSE b/vendor/libmimalloc-sys/c_src/mimalloc/v3/LICENSE
new file mode 100644
index 00000000..53315ebe
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-new-delete.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-new-delete.h
new file mode 100644
index 00000000..c16f4a66
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-new-delete.h
@@ -0,0 +1,66 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MIMALLOC_NEW_DELETE_H
+#define MIMALLOC_NEW_DELETE_H
+
+// ----------------------------------------------------------------------------
+// This header provides convenient overrides for the new and
+// delete operations in C++.
+//
+// This header should be included in only one source file!
+//
+// On Windows, or when linking dynamically with mimalloc, these
+// can be more performant than the standard new-delete operations.
+// See <https://en.cppreference.com/w/cpp/memory/new/operator_new>
+// ---------------------------------------------------------------------------
+#if defined(__cplusplus)
+  #include <new>
+  #include <mimalloc.h>
+
+  #if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_)
+  // stay consistent with VCRT definitions
+  #define mi_decl_new(n)          mi_decl_nodiscard mi_decl_restrict _Ret_notnull_ _Post_writable_byte_size_(n)
+  #define mi_decl_new_nothrow(n)  mi_decl_nodiscard mi_decl_restrict _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n)
+  #else
+  #define mi_decl_new(n)          mi_decl_nodiscard mi_decl_restrict
+  #define mi_decl_new_nothrow(n)  mi_decl_nodiscard mi_decl_restrict
+  #endif
+
+  void operator delete(void* p) noexcept              { mi_free(p); };
+  void operator delete[](void* p) noexcept            { mi_free(p); };
+
+  void operator delete  (void* p, const std::nothrow_t&) noexcept { mi_free(p); }
+  void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }
+
+  mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }
+  mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }
+
+  mi_decl_new_nothrow(n) void* operator new  (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }
+  mi_decl_new_nothrow(n) void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }
+
+  #if (__cplusplus >= 201402L || _MSC_VER >= 1916)
+  void operator delete  (void* p, std::size_t n) noexcept { mi_free_size(p,n); };
+  void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); };
+  #endif
+
+  #if (__cplusplus > 201402L || defined(__cpp_aligned_new))
+  void operator delete  (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  void operator delete  (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
+  void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
+  void operator delete  (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+
+  void* operator new  (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
+  void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
+  void* operator new  (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
+  void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
+  #endif
+#endif
+
+#endif // MIMALLOC_NEW_DELETE_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-override.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-override.h
new file mode 100644
index 00000000..46e05452
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-override.h
@@ -0,0 +1,68 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MIMALLOC_OVERRIDE_H
+#define MIMALLOC_OVERRIDE_H
+
+/* ----------------------------------------------------------------------------
+This header can be used to statically redirect malloc/free and new/delete
+to the mimalloc variants. This can be useful if one can include this file on
+each source file in a project (but be careful when using external code to
+not accidentally mix pointers from different allocators).
+-----------------------------------------------------------------------------*/
+
+#include <mimalloc.h>
+
+// Standard C allocation
+#define malloc(n)               mi_malloc(n)
+#define calloc(c,n)             mi_calloc(c,n)
+#define realloc(p,n)            mi_realloc(p,n)
+#define free(p)                 mi_free(p)
+
+#define strdup(s)               mi_strdup(s)
+#define strndup(s,n)            mi_strndup(s,n)
+#define realpath(f,n)           mi_realpath(f,n)
+
+// Microsoft extensions
+#define _expand(p,n)            mi_expand(p,n)
+#define _msize(p)               mi_usable_size(p)
+#define _recalloc(p,c,n)        mi_recalloc(p,c,n)
+
+#define _strdup(s)              mi_strdup(s)
+#define _strndup(s,n)           mi_strndup(s,n)
+#define _wcsdup(s)              mi_wcsdup(s)
+#define _mbsdup(s)              mi_mbsdup(s)
+#define _dupenv_s(buf,n,nm)     mi_dupenv_s(buf,n,nm)
+#define _wdupenv_s(buf,n,nm)    mi_wdupenv_s(buf,n,nm)
+
+// Various Posix and Unix variants
+#define reallocf(p,n)           mi_reallocf(p,n)
+#define malloc_size(p)          mi_usable_size(p)
+#define malloc_usable_size(p)   mi_usable_size(p)
+#define malloc_good_size(n)     mi_malloc_good_size(n)
+#define cfree(p)                mi_free(p)
+
+#define valloc(n)               mi_valloc(n)
+#define pvalloc(n)              mi_pvalloc(n)
+#define reallocarray(p,c,n)     mi_reallocarray(p,c,n)
+#define reallocarr(ptrp,c,n)    mi_reallocarr(ptrp,c,n)
+#define memalign(a,n)           mi_memalign(a,n)
+#define aligned_alloc(a,n)      mi_aligned_alloc(a,n)
+#define posix_memalign(p,a,n)   mi_posix_memalign(p,a,n)
+#define _posix_memalign(p,a,n)  mi_posix_memalign(p,a,n)
+
+// Microsoft aligned variants
+#define _aligned_malloc(n,a)                  mi_malloc_aligned(n,a)
+#define _aligned_realloc(p,n,a)               mi_realloc_aligned(p,n,a)
+#define _aligned_recalloc(p,c,n,a)            mi_aligned_recalloc(p,c,n,a)
+#define _aligned_msize(p,a,o)                 mi_usable_size(p)
+#define _aligned_free(p)                      mi_free(p)
+#define _aligned_offset_malloc(n,a,o)         mi_malloc_aligned_at(n,a,o)
+#define _aligned_offset_realloc(p,n,a,o)      mi_realloc_aligned_at(p,n,a,o)
+#define _aligned_offset_recalloc(p,c,n,a,o)   mi_recalloc_aligned_at(p,c,n,a,o)
+
+#endif // MIMALLOC_OVERRIDE_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-stats.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-stats.h
new file mode 100644
index 00000000..1c620fd1
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-stats.h
@@ -0,0 +1,168 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2024-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MIMALLOC_STATS_H
+#define MIMALLOC_STATS_H
+
+#include <mimalloc.h>
+#include <string.h>   // memset
+#include <stdint.h>   // int64_t
+
+#define MI_STAT_VERSION   5  // increased on every backward incompatible change
+
+// alignment for atomic fields
+#if defined(_MSC_VER)
+#define mi_decl_align(a)        __declspec(align(a))
+#elif defined(__GNUC__)
+#define mi_decl_align(a)        __attribute__((aligned(a)))
+#elif __cplusplus >= 201103L
+#define mi_decl_align(a)        alignas(a)
+#else
+#define mi_decl_align(a)
+#endif
+
+
+// count allocation over time
+typedef struct mi_stat_count_s {
+  int64_t total;                              // total allocated
+  int64_t peak;                               // peak allocation
+  int64_t current;                            // current allocation
+} mi_stat_count_t;
+
+// counters only increase
+typedef struct mi_stat_counter_s {
+  int64_t total;                              // total count
+} mi_stat_counter_t;
+
+#define MI_STAT_FIELDS() \
+  MI_STAT_COUNT(pages)                      /* count of mimalloc pages */ \
+  MI_STAT_COUNT(reserved)                   /* reserved memory bytes */ \
+  MI_STAT_COUNT(committed)                  /* committed bytes */ \
+  MI_STAT_COUNTER(reset)                    /* reset bytes */ \
+  MI_STAT_COUNTER(purged)                   /* purged bytes */ \
+  MI_STAT_COUNT(page_committed)             /* committed memory inside pages */ \
+  MI_STAT_COUNT(pages_abandoned)            /* abandoned pages count */ \
+  MI_STAT_COUNT(threads)                    /* number of threads */ \
+  MI_STAT_COUNT(malloc_normal)              /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \
+  MI_STAT_COUNT(malloc_huge)                /* allocated bytes in huge pages */ \
+  MI_STAT_COUNT(malloc_requested)           /* malloc requested bytes */ \
+  \
+  MI_STAT_COUNTER(mmap_calls) \
+  MI_STAT_COUNTER(commit_calls) \
+  MI_STAT_COUNTER(reset_calls) \
+  MI_STAT_COUNTER(purge_calls) \
+  MI_STAT_COUNTER(arena_count)              /* number of memory arena's */ \
+  MI_STAT_COUNTER(malloc_normal_count)      /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \
+  MI_STAT_COUNTER(malloc_huge_count)        /* number of huge bloks */ \
+  MI_STAT_COUNTER(malloc_guarded_count)     /* number of allocations with guard pages */ \
+  \
+  /* internal statistics */ \
+  MI_STAT_COUNTER(arena_rollback_count) \
+  MI_STAT_COUNTER(arena_purges) \
+  MI_STAT_COUNTER(pages_extended)           /* number of page extensions */ \
+  MI_STAT_COUNTER(pages_retire)             /* number of pages that are retired */ \
+  MI_STAT_COUNTER(page_searches)            /* total pages searched for a fresh page */ \
+  MI_STAT_COUNTER(page_searches_count)      /* searched count for a fresh page */ \
+  /* only on v1 and v2 */ \
+  MI_STAT_COUNT(segments) \
+  MI_STAT_COUNT(segments_abandoned) \
+  MI_STAT_COUNT(segments_cache) \
+  MI_STAT_COUNT(_segments_reserved) \
+  /* only on v3 */ \
+  MI_STAT_COUNT(heaps) \
+  MI_STAT_COUNT(theaps) \
+  MI_STAT_COUNTER(pages_reclaim_on_alloc) \
+  MI_STAT_COUNTER(pages_reclaim_on_free) \
+  MI_STAT_COUNTER(pages_reabandon_full) \
+  MI_STAT_COUNTER(pages_unabandon_busy_wait) \
+  MI_STAT_COUNTER(heaps_delete_wait)
+
+// Size bins for chunks
+typedef enum mi_chunkbin_e {
+  MI_CBIN_SMALL,    // slice_count == 1
+  MI_CBIN_OTHER,    // slice_count: any other from the other bins, and 1 <= slice_count <= MI_BCHUNK_BITS
+  MI_CBIN_MEDIUM,   // slice_count == 8
+  MI_CBIN_LARGE,    // slice_count == MI_SIZE_BITS  (only used if MI_ENABLE_LARGE_PAGES is 1)
+  MI_CBIN_HUGE,     // slice_count > MI_BCHUNK_BITS
+  MI_CBIN_NONE,     // no bin assigned yet (the chunk is completely free)
+  MI_CBIN_COUNT
+} mi_chunkbin_t;
+
+
+// Define the statistics structure
+#define MI_BIN_HUGE             (73U)   // see types.h
+#define MI_STAT_COUNT(stat)     mi_stat_count_t stat;
+#define MI_STAT_COUNTER(stat)   mi_stat_counter_t stat;
+
+typedef struct mi_stats_s
+{
+  size_t size;          // size of the mi_stats_t structure 
+  size_t version;       
+
+  mi_decl_align(8)  MI_STAT_FIELDS()
+
+  // future extension
+  mi_stat_count_t   _stat_reserved[4];
+  mi_stat_counter_t _stat_counter_reserved[4];
+
+  // size segregated statistics
+  mi_stat_count_t   malloc_bins[MI_BIN_HUGE+1];   // allocation per size bin
+  mi_stat_count_t   page_bins[MI_BIN_HUGE+1];     // pages allocated per size bin
+  mi_stat_count_t   chunk_bins[MI_CBIN_COUNT];    // chunks per page sizes
+} mi_stats_t;
+
+#undef MI_STAT_COUNT
+#undef MI_STAT_COUNTER
+
+// Initialization
+static inline void mi_stats_header_init(mi_stats_t* stats) {
+  stats->size = sizeof(*stats);
+  stats->version = MI_STAT_VERSION;
+}
+static inline void mi_stats_init(mi_stats_t* stats) {
+  memset(stats,0,sizeof(*stats));
+  mi_stats_header_init(stats);
+}
+
+#define mi_stats_t_decl(name)  mi_stats_t name; mi_stats_init(&name);
+
+// Exported definitions
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// stats from a heap
+mi_decl_export bool    mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept;
+mi_decl_export char*   mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept;      // use mi_free to free the result if the input buf == NULL
+mi_decl_export void    mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept;
+
+// stats from a subprocess and its heaps aggregated
+mi_decl_export bool    mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;
+mi_decl_export char*   mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept;      // use mi_free to free the result if the input buf == NULL
+mi_decl_export void    mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;
+// print subprocess and all its heap stats segregated
+mi_decl_export void    mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;
+
+// stats aggregated for the current subprocess and all its heaps.
+mi_decl_export bool    mi_stats_get(mi_stats_t* stats) mi_attr_noexcept;
+mi_decl_export char*   mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept;      // use mi_free to free the result if the input buf == NULL
+mi_decl_export void    mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
+
+// add the stats of the heap to the subprocess and clear the heap stats
+mi_decl_export void    mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
+
+// stats from the subprocess without aggregating its heaps
+mi_decl_export bool    mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;
+
+mi_decl_export char*   mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept;      // use mi_free to free the result if the input buf == NULL
+mi_decl_export size_t  mi_stats_get_bin_size(size_t bin) mi_attr_noexcept;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // MIMALLOC_STATS_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc.h
new file mode 100644
index 00000000..f373f5dc
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc.h
@@ -0,0 +1,706 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MIMALLOC_H
+#define MIMALLOC_H
+
+#define MI_MALLOC_VERSION 30302   // major + 2 digits minor + 2 digits patch
+
+// ------------------------------------------------------
+// Compiler specific attributes
+// ------------------------------------------------------
+
+#ifdef __cplusplus
+  #if (__cplusplus >= 201103L) || (_MSC_VER > 1900)  // C++11
+    #define mi_attr_noexcept   noexcept
+  #else
+    #define mi_attr_noexcept   throw()
+  #endif
+#else
+  #define mi_attr_noexcept
+#endif
+
+#if defined(__cplusplus) && (__cplusplus >= 201703)
+  #define mi_decl_nodiscard    [[nodiscard]]
+#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)  // includes clang, icc, and clang-cl
+  #define mi_decl_nodiscard    __attribute__((warn_unused_result))
+#elif defined(_HAS_NODISCARD)
+  #define mi_decl_nodiscard    _NODISCARD
+#elif (_MSC_VER >= 1700)
+  #define mi_decl_nodiscard    _Check_return_
+#else
+  #define mi_decl_nodiscard
+#endif
+
+#if defined(_MSC_VER) || defined(__MINGW32__)
+  #if !defined(MI_SHARED_LIB)
+    #define mi_decl_export
+  #elif defined(MI_SHARED_LIB_EXPORT)
+    #define mi_decl_export              __declspec(dllexport)
+  #else
+    #define mi_decl_export              __declspec(dllimport)
+  #endif
+  #if defined(__MINGW32__)
+    #define mi_decl_restrict
+    #define mi_attr_malloc              __attribute__((malloc))
+  #else
+    #if (_MSC_VER >= 1900) && !defined(__EDG__)
+      #define mi_decl_restrict          __declspec(allocator) __declspec(restrict)
+    #else
+      #define mi_decl_restrict          __declspec(restrict)
+    #endif
+    #define mi_attr_malloc
+  #endif
+  #define mi_cdecl                      __cdecl
+  #define mi_attr_alloc_size(s)
+  #define mi_attr_alloc_size2(s1,s2)
+  #define mi_attr_alloc_align(p)
+#elif defined(__GNUC__)                 // includes clang and icc
+  #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT)
+    #define mi_decl_export              __attribute__((visibility("default")))
+  #else
+    #define mi_decl_export
+  #endif
+  #define mi_cdecl                      // leads to warnings... __attribute__((cdecl))
+  #define mi_decl_restrict
+  #define mi_attr_malloc                __attribute__((malloc))
+  #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)
+    #define mi_attr_alloc_size(s)
+    #define mi_attr_alloc_size2(s1,s2)
+    #define mi_attr_alloc_align(p)
+  #elif defined(__INTEL_COMPILER)
+    #define mi_attr_alloc_size(s)       __attribute__((alloc_size(s)))
+    #define mi_attr_alloc_size2(s1,s2)  __attribute__((alloc_size(s1,s2)))
+    #define mi_attr_alloc_align(p)
+  #else
+    #define mi_attr_alloc_size(s)       __attribute__((alloc_size(s)))
+    #define mi_attr_alloc_size2(s1,s2)  __attribute__((alloc_size(s1,s2)))
+    #define mi_attr_alloc_align(p)      __attribute__((alloc_align(p)))
+  #endif
+#else
+  #define mi_cdecl
+  #define mi_decl_export
+  #define mi_decl_restrict
+  #define mi_attr_malloc
+  #define mi_attr_alloc_size(s)
+  #define mi_attr_alloc_size2(s1,s2)
+  #define mi_attr_alloc_align(p)
+#endif
+
+// ------------------------------------------------------
+// Includes
+// ------------------------------------------------------
+
+#include <stddef.h>     // size_t
+#include <stdbool.h>    // bool
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// ------------------------------------------------------
+// Standard malloc interface
+// ------------------------------------------------------
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
+mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize)      mi_attr_noexcept mi_attr_alloc_size(2);
+mi_decl_export void* mi_expand(void* p, size_t newsize)                         mi_attr_noexcept mi_attr_alloc_size(2);
+
+mi_decl_export void mi_free(void* p) mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept;
+
+// ------------------------------------------------------
+// Extended allocation functions
+// ------------------------------------------------------
+#define MI_SMALL_WSIZE_MAX  (128)
+#define MI_SMALL_SIZE_MAX   (MI_SMALL_WSIZE_MAX*sizeof(void*))
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size)       mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
+mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size)        mi_attr_noexcept mi_attr_alloc_size2(2,3);
+mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize)                   mi_attr_noexcept mi_attr_alloc_size(2);
+
+mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size)     mi_attr_noexcept;
+
+// `mi_free_small` is for special applications like language runtimes.
+// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free`
+mi_decl_export void mi_free_small(void* p) mi_attr_noexcept;  
+
+// -------------------------------------------------------------------------------------
+// Aligned allocation
+// Note that `alignment` always follows `size` for consistency with unaligned
+// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`.
+// -------------------------------------------------------------------------------------
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2) mi_attr_alloc_align(3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2);
+mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
+mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
+
+
+// ------------------------------------------------------
+// Typed allocation, the type is always the first parameter
+// ------------------------------------------------------
+
+#define mi_malloc_tp(tp)                 ((tp*)mi_malloc(sizeof(tp)))
+#define mi_zalloc_tp(tp)                 ((tp*)mi_zalloc(sizeof(tp)))
+#define mi_calloc_tp(tp,n)               ((tp*)mi_calloc(n,sizeof(tp)))
+#define mi_mallocn_tp(tp,n)              ((tp*)mi_mallocn(n,sizeof(tp)))
+#define mi_reallocn_tp(tp,p,n)           ((tp*)mi_reallocn(p,n,sizeof(tp)))
+#define mi_recalloc_tp(tp,p,n)           ((tp*)mi_recalloc(p,n,sizeof(tp)))
+
+#define mi_heap_malloc_tp(tp,hp)         ((tp*)mi_heap_malloc(hp,sizeof(tp)))
+#define mi_heap_zalloc_tp(tp,hp)         ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
+#define mi_heap_calloc_tp(tp,hp,n)       ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))
+#define mi_heap_mallocn_tp(tp,hp,n)      ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))
+#define mi_heap_reallocn_tp(tp,hp,p,n)   ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp)))
+#define mi_heap_recalloc_tp(tp,hp,p,n)   ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp)))
+
+
+// ------------------------------------------------------
+// Internals
+// See also `mimalloc-stats.h` for statistics
+// ------------------------------------------------------
+
+typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
+mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept;
+
+typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg);
+mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept;
+
+typedef void (mi_cdecl mi_error_fun)(int err, void* arg);
+mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);
+
+mi_decl_export void mi_collect(bool force)      mi_attr_noexcept;
+mi_decl_export int  mi_version(void)            mi_attr_noexcept;
+mi_decl_export void mi_options_print(void)      mi_attr_noexcept;
+mi_decl_export void mi_process_info_print(void) mi_attr_noexcept;
+mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg)      mi_attr_noexcept;
+mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
+mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs,
+                                    size_t* current_rss, size_t* peak_rss,
+                                    size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept;
+
+
+
+// Generally do not use the following as these are usually called automatically
+mi_decl_export void mi_process_init(void)     mi_attr_noexcept;
+mi_decl_export void mi_cdecl mi_process_done(void) mi_attr_noexcept;
+mi_decl_export void mi_thread_init(void)      mi_attr_noexcept;
+mi_decl_export void mi_thread_done(void)      mi_attr_noexcept;
+mi_decl_export void mi_thread_set_in_threadpool(void) mi_attr_noexcept; // communicate that a thread is in a threadpool
+
+
+// -----------------------------------------------------------------
+// Return allocated block size (if the return value is not NULL)
+// -----------------------------------------------------------------
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc(size_t size, size_t* block_size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* block_size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
+mi_decl_nodiscard mi_decl_export void* mi_urealloc(void* p, size_t newsize, size_t* block_size_pre, size_t* block_size_post) mi_attr_noexcept mi_attr_alloc_size(2);
+mi_decl_export void mi_ufree(void* p, size_t* block_size) mi_attr_noexcept;
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+
+
+// -------------------------------------------------------------------------------------
+// Heaps: first-class. Can allocate from any thread (and be free'd from any thread)
+// Heaps keep allocations in separate pages from each other (but share the arena's and free'd pages)
+// -------------------------------------------------------------------------------------
+
+struct mi_heap_s;
+typedef struct mi_heap_s mi_heap_t;
+
+mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void);
+mi_decl_export void mi_heap_delete(mi_heap_t* heap);            // move live blocks to the main heap
+mi_decl_export void mi_heap_destroy(mi_heap_t* heap);           // free all live blocks
+mi_decl_export void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
+mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force);
+
+mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_main(void);
+mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_of(const void* p);
+mi_decl_nodiscard mi_decl_export bool       mi_heap_contains(const mi_heap_t* heap, const void* p);
+mi_decl_nodiscard mi_decl_export bool       mi_any_heap_contains(const void* p);
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+
+mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize)              mi_attr_noexcept mi_attr_alloc_size(3);
+mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size)  mi_attr_noexcept mi_attr_alloc_size2(3, 4);
+mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* theap, void* p, size_t newsize)            mi_attr_noexcept mi_attr_alloc_size(3);
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s)            mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept;
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
+mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
+mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
+
+
+// --------------------------------------------------------------------------------
+// Zero initialized re-allocation.
+// Only valid on memory that was originally allocated with zero initialization too.
+// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.
+// see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992>
+// --------------------------------------------------------------------------------
+
+mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize)                mi_attr_noexcept mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size)  mi_attr_noexcept mi_attr_alloc_size2(2,3);
+
+mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
+mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4);
+mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3);
+
+mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize)                mi_attr_noexcept mi_attr_alloc_size(3);
+mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size)  mi_attr_noexcept mi_attr_alloc_size2(3, 4);
+
+mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
+mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
+mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5);
+mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
+
+
+
+// ------------------------------------------------------
+// Visiting pages and individual blocks in a heap.
+// ------------------------------------------------------
+
+// An area of heap space contains blocks of a single size.
+typedef struct mi_heap_area_s {
+  void*  blocks;      // start of the area containing theap blocks
+  size_t reserved;    // bytes reserved for this area (virtual)
+  size_t committed;   // current available bytes for this area
+  size_t used;        // number of allocated blocks
+  size_t block_size;  // size in bytes of each block
+  size_t full_block_size; // size in bytes of a full block including padding and metadata.
+  void*  reserved1;   // internal
+} mi_heap_area_t;
+
+typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
+
+mi_decl_export bool   mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
+mi_decl_export bool   mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
+
+
+// ------------------------------------------------------
+// Arena memory management
+// Arena's are larger memory area's provided by the OS or user
+// ------------------------------------------------------
+
+mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;
+
+mi_decl_export int    mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;
+mi_decl_export int    mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;
+
+mi_decl_export int    mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
+mi_decl_export bool   mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_pinned /* cannot decommit/reset? */, bool is_zero, int numa_node) mi_attr_noexcept;
+
+mi_decl_export void   mi_debug_show_arenas(void) mi_attr_noexcept;
+mi_decl_export void   mi_arenas_print(void) mi_attr_noexcept;
+mi_decl_export size_t mi_arena_min_alignment(void);
+mi_decl_export size_t mi_arena_min_size(void);
+
+typedef void* mi_arena_id_t;
+mi_decl_export void*  mi_arena_area(mi_arena_id_t arena_id, size_t* size);
+mi_decl_export int    mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
+mi_decl_export int    mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
+mi_decl_export bool   mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
+mi_decl_export bool   mi_arena_contains(mi_arena_id_t arena_id, const void* p);
+
+// Create a heap that only allocates in the specified arena
+mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
+
+
+// ------------------------------------------------------
+// Subprocesses
+// Advanced: allow sub-processes whose memory arena's stay fully separated (and no reclamation between them).
+// Used for example for separate interpreters in one process.
+// ------------------------------------------------------
+
+typedef struct { void* _mi_subproc_id; } mi_subproc_id_t;  // abstract type
+mi_decl_export mi_subproc_id_t mi_subproc_main(void);
+mi_decl_export mi_subproc_id_t mi_subproc_current(void);
+mi_decl_export mi_subproc_id_t mi_subproc_new(void);
+mi_decl_export void mi_subproc_destroy(mi_subproc_id_t subproc);
+mi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet)
+
+typedef bool (mi_cdecl mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
+mi_decl_export bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
+
+
+// -------------------------------------------------------------------------------------
+// A "theap" is a thread-local heap. This API is only provided for special circumstances like runtimes
+// that already have a thread-local context and can store the theap there for (slightly) faster allocations.
+// This also allows to set a default theap for the current thread so that `malloc` etc. allocate from
+// that theap (instead of the main (t)heap).
+// Theaps are first-class, but can only allocate from the same thread that created it.
+// Allocation through a `theap` may be a tiny bit faster than using plain malloc
+// (as we don't need to lookup the thread local variable).
+// -------------------------------------------------------------------------------------
+
+struct mi_theap_s;
+typedef struct mi_theap_s mi_theap_t;
+
+mi_decl_export mi_theap_t* mi_heap_theap(mi_heap_t* heap);
+mi_decl_export mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
+mi_decl_export mi_theap_t* mi_theap_get_default(void);
+mi_decl_export void        mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept;
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
+mi_decl_nodiscard mi_decl_export                  void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize)              mi_attr_noexcept mi_attr_alloc_size(3);
+
+
+// ------------------------------------------------------
+// Experimental
+// ------------------------------------------------------
+
+// Experimental: objects followed by a guard page.
+// Setting the sample rate on a specific theap can be used to test parts of the program more
+// specifically (in combination with `mi_theap_set_default`).
+// A sample rate of 0 disables guarded objects, while 1 uses a guard page for every object.
+// A seed of 0 uses a random start point. Only objects within the size bound are eligable for guard pages.
+mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed);
+mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max);
+
+// very experimental
+typedef bool (mi_cdecl mi_commit_fun_t)(bool commit, void* start, size_t size, bool* is_zero, void* user_arg);
+mi_decl_export bool  mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive,
+                                      mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept;
+
+//mi_decl_export bool  mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* size);
+//mi_decl_export bool  mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id);
+//mi_decl_export bool  mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena);
+//mi_decl_export void  mi_theap_unload(mi_theap_t* theap);
+
+// unsafe: assumes the page belonging to `p` is only accessed by the calling thread.
+mi_decl_export bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept;
+
+// ------------------------------------------------------
+// Deprecated
+// ------------------------------------------------------
+
+mi_decl_export bool mi_check_owned(const void* p);
+
+mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
+mi_decl_export bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
+
+mi_decl_export int  mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
+mi_decl_export void mi_collect_reduce(size_t target_thread_owned) mi_attr_noexcept;
+
+mi_decl_export void mi_stats_reset(void)      mi_attr_noexcept;
+mi_decl_export void mi_stats_merge(void)      mi_attr_noexcept;
+mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept;  // backward compatibility: `out` is ignored and should be NULL
+
+mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;  // not deprecated but declared in `mimalloc-stats.h` now.
+
+
+// ------------------------------------------------------
+// Options
+// ------------------------------------------------------
+
+typedef enum mi_option_e {
+  // stable options
+  mi_option_show_errors,                // print error messages
+  mi_option_show_stats,                 // print statistics on termination
+  mi_option_verbose,                    // print verbose messages
+  // advanced options
+  mi_option_deprecated_eager_commit,    
+  mi_option_arena_eager_commit,         // eager commit arenas? Use 2 to enable just on overcommit systems (=2)
+  mi_option_purge_decommits,            // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
+  mi_option_allow_large_os_pages,       // allow use of large (2 or 4 MiB) OS pages, implies eager commit.
+  mi_option_reserve_huge_os_pages,      // reserve N huge OS pages (1GiB pages) at startup
+  mi_option_reserve_huge_os_pages_at,   // reserve huge OS pages at a specific NUMA node
+  mi_option_reserve_os_memory,          // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`)
+  mi_option_deprecated_segment_cache,
+  mi_option_deprecated_page_reset,
+  mi_option_deprecated_abandoned_page_purge,
+  mi_option_deprecated_segment_reset,
+  mi_option_deprecated_eager_commit_delay, 
+  mi_option_purge_delay,                // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10)
+  mi_option_use_numa_nodes,             // 0 = use all available numa nodes, otherwise use at most N nodes.
+  mi_option_disallow_os_alloc,          // 1 = do not use OS memory for allocation (but only programmatically reserved arenas)
+  mi_option_os_tag,                     // tag used for OS logging (macOS only for now) (=100)
+  mi_option_max_errors,                 // issue at most N error messages
+  mi_option_max_warnings,               // issue at most N warning messages
+  mi_option_deprecated_max_segment_reclaim,  // max. percentage of the abandoned segments can be reclaimed per try (=10%)
+  mi_option_destroy_on_exit,            // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
+  mi_option_arena_reserve,              // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`)
+  mi_option_arena_purge_mult,           // multiplier for `purge_delay` for the purging delay for arenas (=10)
+  mi_option_deprecated_purge_extend_delay,
+  mi_option_disallow_arena_alloc,       // 1 = do not use arena's for allocation (except if using specific arena id's)
+  mi_option_retry_on_oom,               // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
+  mi_option_visit_abandoned,            // allow visiting theap blocks from abandoned threads (=0)
+  mi_option_guarded_min,                // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
+  mi_option_guarded_max,                // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
+  mi_option_guarded_precise,            // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
+  mi_option_guarded_sample_rate,        // 1 out of N allocations in the min/max range will be guarded (=1000)
+  mi_option_guarded_sample_seed,        // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
+  mi_option_generic_collect,            // collect theaps every N (=10000) generic allocation calls
+  mi_option_page_reclaim_on_free,       // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always
+  mi_option_page_full_retain,           // retain N full (small) pages per size class (=2)
+  mi_option_page_max_candidates,        // max candidate pages to consider for allocation (=4)
+  mi_option_max_vabits,                 // max user space virtual address bits to consider (=48)
+  mi_option_pagemap_commit,             // commit the full pagemap (to always catch invalid pointer uses) (=0)
+  mi_option_page_commit_on_demand,      // commit page memory on-demand
+  mi_option_page_max_reclaim,           // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited))
+  mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16)
+  mi_option_allow_thp,                  // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.
+  mi_option_minimal_purge_size,         // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled.
+  mi_option_arena_max_object_size,      // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). 
+  mi_option_arena_is_numa_local,        // experimental
+  _mi_option_last,
+  // legacy option names
+  mi_option_large_os_pages = mi_option_allow_large_os_pages,
+  mi_option_eager_region_commit = mi_option_arena_eager_commit,
+  mi_option_reset_decommits = mi_option_purge_decommits,
+  mi_option_reset_delay = mi_option_purge_delay,
+  mi_option_limit_os_alloc = mi_option_disallow_os_alloc
+} mi_option_t;
+
+
+mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);
+mi_decl_export void mi_option_enable(mi_option_t option);
+mi_decl_export void mi_option_disable(mi_option_t option);
+mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
+mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
+
+mi_decl_nodiscard mi_decl_export long   mi_option_get(mi_option_t option);
+mi_decl_nodiscard mi_decl_export long   mi_option_get_clamp(mi_option_t option, long min, long max);
+mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option);
+mi_decl_export void mi_option_set(mi_option_t option, long value);
+mi_decl_export void mi_option_set_default(mi_option_t option, long value);
+
+
+// -------------------------------------------------------------------------------------------------------
+// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.
+// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)
+// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing.
+// -------------------------------------------------------------------------------------------------------
+
+mi_decl_export void  mi_cfree(void* p) mi_attr_noexcept;
+mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p)        mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size)     mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
+
+mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size)   mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
+
+mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
+mi_decl_nodiscard mi_decl_export int   mi_reallocarr(void* ptrp, size_t count, size_t size) mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;
+
+mi_decl_export void mi_free_size(void* p, size_t size)                           mi_attr_noexcept;
+mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;
+mi_decl_export void mi_free_aligned(void* p, size_t alignment)                   mi_attr_noexcept;
+mi_decl_export int  mi_dupenv_s(char** buf, size_t* size, const char* name)      mi_attr_noexcept;
+
+// wide characters
+#include <wchar.h>  // wchar_t
+mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name)       mi_attr_noexcept;
+mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s)  mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s)  mi_attr_noexcept mi_attr_malloc;
+
+// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
+// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size)                   mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size)           mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size)   mi_attr_malloc mi_attr_alloc_size2(1, 2);
+mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize)                mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
+
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size)                 mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
+
+#ifdef __cplusplus
+}
+#endif
+
+// ---------------------------------------------------------------------------------------------
+// Implement the C++ std::allocator interface for use in STL containers.
+// (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally)
+// ---------------------------------------------------------------------------------------------
+#ifdef __cplusplus
+
+#include <cstddef>     // std::size_t
+#include <cstdint>     // PTRDIFF_MAX
+#if (__cplusplus >= 201103L) || (_MSC_VER > 1900)  // C++11
+#include <type_traits> // std::true_type
+#include <utility>     // std::forward
+#endif
+
+template<class T> struct _mi_stl_allocator_common {
+  typedef T                 value_type;
+  typedef std::size_t       size_type;
+  typedef std::ptrdiff_t    difference_type;
+  typedef value_type&       reference;
+  typedef value_type const& const_reference;
+  typedef value_type*       pointer;
+  typedef value_type const* const_pointer;
+
+  #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900))  // C++11
+  using propagate_on_container_copy_assignment = std::true_type;
+  using propagate_on_container_move_assignment = std::true_type;
+  using propagate_on_container_swap            = std::true_type;
+  template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); }
+  template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); }
+  #else
+  void construct(pointer p, value_type const& val) { ::new(p) value_type(val); }
+  void destroy(pointer p) { p->~value_type(); }
+  #endif
+
+  size_type     max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); }
+  pointer       address(reference x) const        { return &x; }
+  const_pointer address(const_reference x) const  { return &x; }
+};
+
+template<class T> struct mi_stl_allocator : public _mi_stl_allocator_common<T> {
+  using typename _mi_stl_allocator_common<T>::size_type;
+  using typename _mi_stl_allocator_common<T>::value_type;
+  using typename _mi_stl_allocator_common<T>::pointer;
+  template <class U> struct rebind { typedef mi_stl_allocator<U> other; };
+
+  mi_stl_allocator()                                             mi_attr_noexcept = default;
+  mi_stl_allocator(const mi_stl_allocator&)                      mi_attr_noexcept = default;
+  template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { }
+  mi_stl_allocator  select_on_container_copy_construction() const { return *this; }
+  void              deallocate(T* p, size_type) { mi_free(p); }
+
+  #if (__cplusplus >= 201703L)  // C++17
+  mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); }
+  mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
+  #else
+  mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); }
+  #endif
+
+  #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900))  // C++11
+  using is_always_equal = std::true_type;
+  #endif
+};
+
+template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; }
+template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; }
+
+
+#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900)  // C++11
+#define MI_HAS_HEAP_STL_ALLOCATOR 1
+
+#include <memory>      // std::shared_ptr
+
+// Common base class for STL allocators in a specific theap
+template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> {
+  using typename _mi_stl_allocator_common<T>::size_type;
+  using typename _mi_stl_allocator_common<T>::value_type;
+  using typename _mi_stl_allocator_common<T>::pointer;
+
+  _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp, [](mi_heap_t*) {}) {}    /* will not delete nor destroy the passed in heap */
+
+  #if (__cplusplus >= 201703L)  // C++17
+  mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); }
+  mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
+  #else
+  mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); }
+  #endif
+
+  #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900))  // C++11
+  using is_always_equal = std::false_type;
+  #endif
+
+  void collect(bool force) { mi_heap_collect(this->heap.get(), force); }
+  template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); }
+
+protected:
+  std::shared_ptr<mi_heap_t> heap;
+  template<class U, bool D> friend struct _mi_heap_stl_allocator_common;
+
+  _mi_heap_stl_allocator_common() {
+    mi_heap_t* hp = mi_heap_new();
+    this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete));  /* calls heap_delete/destroy when the refcount drops to zero */
+  }
+  _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }
+  template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { }
+
+private:
+  static void heap_delete(mi_heap_t* hp)  { if (hp != NULL) { mi_heap_delete(hp); } }
+  static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } }
+};
+
+// STL allocator allocation in a specific heap
+template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> {
+  using typename _mi_heap_stl_allocator_common<T, false>::size_type;
+  mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called
+  mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { }  // no delete nor destroy on the passed in heap
+  template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { }
+
+  mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; }
+  void deallocate(T* p, size_type) { mi_free(p); }
+  template<class U> struct rebind { typedef mi_heap_stl_allocator<U> other; };
+};
+
+template<class T1, class T2> bool operator==(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
+template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
+
+
+// STL allocator allocation in a specific heap, where `free` does nothing and
+// the heap is destroyed in one go on destruction -- use with care!
+template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> {
+  using typename _mi_heap_stl_allocator_common<T, true>::size_type;
+  mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called
+  mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { }  // no delete nor destroy on the passed in heap
+  template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { }
+
+  mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; }
+  void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ }
+  template<class U> struct rebind { typedef mi_heap_destroy_stl_allocator<U> other; };
+};
+
+template<class T1, class T2> bool operator==(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
+template<class T1, class T2> bool operator!=(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
+
+#endif // C++11
+
+#endif // __cplusplus
+
+#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/atomic.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/atomic.h
new file mode 100644
index 00000000..e4abde6b
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/atomic.h
@@ -0,0 +1,562 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2024 Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MI_ATOMIC_H
+#define MI_ATOMIC_H
+
+// include windows.h or pthreads.h
+#if defined(_WIN32)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))
+#define  MI_USE_PTHREADS
+#include <pthread.h>
+#endif
+
+// --------------------------------------------------------------------------------------------
+// Atomics
+// We need to be portable between C, C++, and MSVC.
+// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode.
+// This is why we try to use only `uintptr_t` and `<type>*` as atomic types.
+// To gain better insight in the range of used atomics, we use explicitly named memory order operations
+// instead of passing the memory order as a parameter.
+// -----------------------------------------------------------------------------------------------
+
+#if defined(__cplusplus)
+// Use C++ atomics
+#include <atomic>
+#define  _Atomic(tp)              std::atomic<tp>
+#define  mi_atomic(name)          std::atomic_##name
+#define  mi_memory_order(name)    std::memory_order_##name
+#if (__cplusplus >= 202002L)      // c++20, see issue #571
+ #define MI_ATOMIC_VAR_INIT(x)    x
+#elif !defined(ATOMIC_VAR_INIT)
+ #define MI_ATOMIC_VAR_INIT(x)    x
+#else
+ #define MI_ATOMIC_VAR_INIT(x)    ATOMIC_VAR_INIT(x)
+#endif
+#elif defined(_MSC_VER)
+// Use MSVC C wrapper for C11 atomics
+#define  _Atomic(tp)              tp
+#define  MI_ATOMIC_VAR_INIT(x)    x
+#define  mi_atomic(name)          mi_atomic_##name
+#define  mi_memory_order(name)    mi_memory_order_##name
+#else
+// Use C11 atomics
+#include <stdatomic.h>
+#define  mi_atomic(name)          atomic_##name
+#define  mi_memory_order(name)    memory_order_##name
+#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735
+ #define MI_ATOMIC_VAR_INIT(x)    x
+#elif !defined(ATOMIC_VAR_INIT)
+ #define MI_ATOMIC_VAR_INIT(x)    x
+#else
+ #define MI_ATOMIC_VAR_INIT(x)    ATOMIC_VAR_INIT(x)
+#endif
+#endif
+
+// Various defines for all used memory orders in mimalloc
+#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail)  \
+  mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail)
+
+#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail)  \
+  mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail)
+
+#define mi_atomic_load_acquire(p)                mi_atomic(load_explicit)(p,mi_memory_order(acquire))
+#define mi_atomic_load_relaxed(p)                mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
+#define mi_atomic_store_release(p,x)             mi_atomic(store_explicit)(p,x,mi_memory_order(release))
+#define mi_atomic_store_relaxed(p,x)             mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
+#define mi_atomic_exchange_relaxed(p,x)          mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed))
+#define mi_atomic_exchange_release(p,x)          mi_atomic(exchange_explicit)(p,x,mi_memory_order(release))
+#define mi_atomic_exchange_acq_rel(p,x)          mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel))
+
+#define mi_atomic_cas_weak_relaxed(p,exp,des)    mi_atomic_cas_weak(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))
+#define mi_atomic_cas_weak_release(p,exp,des)    mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
+#define mi_atomic_cas_weak_acq_rel(p,exp,des)    mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
+#define mi_atomic_cas_strong_relaxed(p,exp,des)  mi_atomic_cas_strong(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))
+#define mi_atomic_cas_strong_release(p,exp,des)  mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
+#define mi_atomic_cas_strong_acq_rel(p,exp,des)  mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
+
+#define mi_atomic_add_relaxed(p,x)               mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed))
+#define mi_atomic_add_acq_rel(p,x)               mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel))
+#define mi_atomic_sub_relaxed(p,x)               mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed))
+#define mi_atomic_sub_acq_rel(p,x)               mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel))
+#define mi_atomic_and_relaxed(p,x)               mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(relaxed))
+#define mi_atomic_and_acq_rel(p,x)               mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel))
+#define mi_atomic_or_relaxed(p,x)                mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(relaxed))
+#define mi_atomic_or_acq_rel(p,x)                mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel))
+
+#define mi_atomic_increment_relaxed(p)           mi_atomic_add_relaxed(p,(uintptr_t)1)
+#define mi_atomic_decrement_relaxed(p)           mi_atomic_sub_relaxed(p,(uintptr_t)1)
+#define mi_atomic_increment_acq_rel(p)           mi_atomic_add_acq_rel(p,(uintptr_t)1)
+#define mi_atomic_decrement_acq_rel(p)           mi_atomic_sub_acq_rel(p,(uintptr_t)1)
+
+static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add);
+static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);
+
+
+#if defined(__cplusplus) || !defined(_MSC_VER)
+
+// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value)
+// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well
+#define mi_atomic_load_ptr_acquire(tp,p)                mi_atomic_load_acquire(p)
+#define mi_atomic_load_ptr_relaxed(tp,p)                mi_atomic_load_relaxed(p)
+
+// In C++ we need to add casts to help resolve templates if NULL is passed
+#if defined(__cplusplus)
+#define mi_atomic_store_ptr_release(tp,p,x)             mi_atomic_store_release(p,(tp*)x)
+#define mi_atomic_store_ptr_relaxed(tp,p,x)             mi_atomic_store_relaxed(p,(tp*)x)
+#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des)    mi_atomic_cas_weak_release(p,exp,(tp*)des)
+#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des)    mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des)
+#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des)  mi_atomic_cas_strong_release(p,exp,(tp*)des)
+#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des)  mi_atomic_cas_strong_acq_rel(p,exp,(tp*)des)
+#define mi_atomic_exchange_ptr_relaxed(tp,p,x)          mi_atomic_exchange_relaxed(p,(tp*)x)
+#define mi_atomic_exchange_ptr_release(tp,p,x)          mi_atomic_exchange_release(p,(tp*)x)
+#define mi_atomic_exchange_ptr_acq_rel(tp,p,x)          mi_atomic_exchange_acq_rel(p,(tp*)x)
+#else
+#define mi_atomic_store_ptr_release(tp,p,x)             mi_atomic_store_release(p,x)
+#define mi_atomic_store_ptr_relaxed(tp,p,x)             mi_atomic_store_relaxed(p,x)
+#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des)    mi_atomic_cas_weak_release(p,exp,des)
+#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des)    mi_atomic_cas_weak_acq_rel(p,exp,des)
+#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des)  mi_atomic_cas_strong_release(p,exp,des)
+#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des)  mi_atomic_cas_strong_acq_rel(p,exp,des)
+#define mi_atomic_exchange_ptr_relaxed(tp,p,x)          mi_atomic_exchange_relaxed(p,x)
+#define mi_atomic_exchange_ptr_release(tp,p,x)          mi_atomic_exchange_release(p,x)
+#define mi_atomic_exchange_ptr_acq_rel(tp,p,x)          mi_atomic_exchange_acq_rel(p,x)
+#endif
+
+// These are used by the statistics
+static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) {
+  return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
+}
+static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
+  const int64_t add = mi_atomic_load_relaxed((_Atomic(int64_t)*)padd);
+  if (add != 0) {
+    mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
+  }
+}
+static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
+  int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p);
+  while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, &current, x)) { /* nothing */ };
+}
+
+// Used by timers
+#define mi_atomic_loadi64_acquire(p)            mi_atomic(load_explicit)(p,mi_memory_order(acquire))
+#define mi_atomic_loadi64_relaxed(p)            mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
+#define mi_atomic_storei64_release(p,x)         mi_atomic(store_explicit)(p,x,mi_memory_order(release))
+#define mi_atomic_storei64_relaxed(p,x)         mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
+
+#define mi_atomic_casi64_strong_acq_rel(p,e,d)  mi_atomic_cas_strong_acq_rel(p,e,d)
+#define mi_atomic_addi64_acq_rel(p,i)           mi_atomic_add_acq_rel(p,i)
+
+
+#elif defined(_MSC_VER)
+
+// Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
+// It is recommended to always compile as C++ when using MSVC.
+
+#include <intrin.h>
+#ifdef _WIN64
+typedef LONG64        msc_intptr_t;
+#define MI_MSC_64(f)  f##64
+#define MI_MSC_XX(f)  f##64
+#else
+typedef LONG          msc_intptr_t;
+#define MI_MSC_64(f)  f
+#define MI_MSC_XX(f)  f##32
+#endif
+
+typedef enum mi_memory_order_e {
+  mi_memory_order_relaxed,
+  mi_memory_order_consume,
+  mi_memory_order_acquire,
+  mi_memory_order_release,
+  mi_memory_order_acq_rel,
+  mi_memory_order_seq_cst
+} mi_memory_order;
+
+static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) {
+  (void)(mo);
+  return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);
+}
+static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) {
+  (void)(mo);
+  return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub));
+}
+static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
+  (void)(mo);
+  return (uintptr_t)MI_MSC_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
+}
+static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
+  (void)(mo);
+  return (uintptr_t)MI_MSC_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
+}
+static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
+  (void)(mo1); (void)(mo2);
+  const uintptr_t read = (uintptr_t)MI_MSC_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected));
+  if (read == *expected) {
+    return true;
+  }
+  else {
+    *expected = read;
+    return false;
+  }
+}
+static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
+  return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2);
+}
+static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) {
+  (void)(mo);
+  return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
+}
+static inline void mi_atomic_thread_fence(mi_memory_order mo) {
+  (void)(mo);
+  _Atomic(uintptr_t) x = 0;
+  mi_atomic_exchange_explicit(&x, 1, mo);
+}
+
+static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {
+  (void)(mo);
+  // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
+  #if defined(_M_IX86) || defined(_M_X64)
+    return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
+  #elif defined(_M_ARM) || defined(_M_ARM64)
+    if (mo == mi_memory_order_relaxed) {
+      return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
+    }
+    else if (mo <= mi_memory_order_acquire) {
+      return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p);
+    }
+    else {
+      const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
+      __dmb(15);  // _ARM(64)_BARRIER_SY
+      return u;
+    }
+  #else
+    #warning "define mi_atomic_load_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
+    return MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
+  #endif
+}
+static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
+  (void)(mo);
+  // assert(mo<=mi_memory_order_release); // others are not used by mimalloc
+  #if defined(_M_IX86) || defined(_M_X64)
+    MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
+  #elif defined(_M_ARM) || defined(_M_ARM64)
+    if (mo == mi_memory_order_relaxed) {
+      MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
+    }
+    else if (mo <= mi_memory_order_release) {
+      MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x);
+    }
+    else {
+      mi_atomic_exchange_explicit(p, x, mo);
+    }
+  #else
+    mi_atomic_exchange_explicit(p, x, mo);
+  #endif
+}
+
+static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) {
+  (void)(mo);
+  // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
+  #if defined(_M_IX86) || defined(_M_X64)
+    return __iso_volatile_load64((volatile const int64_t*)p);
+  #elif defined(_M_ARM) || defined(_M_ARM64)
+    if (mo == mi_memory_order_relaxed) {
+      return __iso_volatile_load64((volatile const int64_t*)p);
+    }
+    #if defined(_M_ARM64)
+    else if (mo <= mi_memory_order_acquire) {
+      return __ldar64((volatile const uintptr_t*)p);
+    }
+    #endif
+    else {
+      const int64_t i = __iso_volatile_load64((volatile const int64_t*)p);
+      __dmb(15);  // _ARM(64)_BARRIER_SY
+      return i;
+    }
+  #else
+    #warning "define mi_atomic_loadi64_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
+    return __iso_volatile_load64((volatile const int64_t*)p);
+  #endif
+}
+
+static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) {
+  (void)(mo);
+  // assert(mo<=mi_memory_order_release); // others are not used by mimalloc
+  #if defined(_M_IX86) || defined(_M_X64)
+    __iso_volatile_store64((volatile int64_t*)p,x);
+  #elif defined(_M_ARM) || defined(_M_ARM64)
+    if (mo == mi_memory_order_relaxed) {
+      __iso_volatile_store64((volatile int64_t*)p,x);
+    }
+    #if defined(_M_ARM64)
+    else if (mo == mi_memory_order_release) {
+      __stlr64((volatile uint64_t*)p, (uint64_t)x);
+    }
+    #endif
+    else {
+      InterlockedExchange64(p, x);
+    }
+  #else
+    InterlockedExchange64(p, x);
+  #endif
+}
+
+// These are used by the statistics
+static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) {
+  #ifdef _WIN64
+    return (int64_t)mi_atomic_addi((int64_t*)p, add);
+  #elif defined(_M_ARM)
+    return _InterlockedExchangeAdd64(p, add);
+  #else
+    // x86
+    int64_t current;
+    int64_t sum;
+    do {
+      current = __iso_volatile_load64((volatile const int64_t*)p);
+      sum = current + add;
+    } while (_InterlockedCompareExchange64(p, sum, current) != current);
+    return current;
+  #endif
+}
+
+static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
+  const int64_t add = *padd;
+  if (add != 0) {
+    mi_atomic_addi64_relaxed((volatile _Atomic(int64_t)*)p, add);
+  }
+}
+
+static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) {
+  int64_t current;
+  do {
+    current = *p;
+  } while (current < x && _InterlockedCompareExchange64(p, x, current) != current);
+}
+
+static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {
+  mi_atomic_addi64_relaxed(p, i);
+}
+
+static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {
+  const int64_t read = _InterlockedCompareExchange64(p, des, *exp);
+  if (read == *exp) {
+    return true;
+  }
+  else {
+    *exp = read;
+    return false;
+  }
+}
+
+// The pointer macros cast to `uintptr_t`.
+#define mi_atomic_load_ptr_acquire(tp,p)                (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))
+#define mi_atomic_load_ptr_relaxed(tp,p)                (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))
+#define mi_atomic_store_ptr_release(tp,p,x)             mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
+#define mi_atomic_store_ptr_relaxed(tp,p,x)             mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
+#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des)    mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
+#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des)    mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
+#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des)  mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
+#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des)  mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
+#define mi_atomic_exchange_ptr_relaxed(tp,p,x)          (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
+#define mi_atomic_exchange_ptr_release(tp,p,x)          (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
+#define mi_atomic_exchange_ptr_acq_rel(tp,p,x)          (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
+
+#define mi_atomic_loadi64_acquire(p)    mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire))
+#define mi_atomic_loadi64_relaxed(p)    mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed))
+#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release))
+#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed))
+
+
+#endif
+
+
+// Atomically add a signed value; returns the previous value.
+static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) {
+  return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add);
+}
+
+// Atomically subtract a signed value; returns the previous value.
+static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {
+  return (intptr_t)mi_atomic_addi(p, -sub);
+}
+
+
+// ----------------------------------------------------------------------
+// Guard
+// ----------------------------------------------------------------------
+
+typedef _Atomic(uintptr_t) mi_atomic_guard_t;
+
+// Allows only one thread to execute at a time (without blocking anyone)
+#define mi_atomic_guard(guard) \
+  uintptr_t _mi_guard_expected = 0; \
+  for(bool _mi_guard_once = true; \
+      _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \
+      (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )
+
+
+// ----------------------------------------------------------------------
+// Locks
+// These should be light-weight in-process only locks.
+// Only used for reserving arena's and to maintain the abandoned list.
+// ----------------------------------------------------------------------
+#if _MSC_VER
+#pragma warning(disable:26110)  // unlock with holding lock
+#endif
+
+#define mi_lock(lock)                  for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) )
+#define mi_lock_maybe(lock,acquire)    for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )
+
+
+#if defined(_WIN32)
+
+typedef struct mi_lock_s {
+  SRWLOCK mutex;    // slim reader-writer lock
+} mi_lock_t;
+
+#define MI_LOCK_INITIALIZER   { SRWLOCK_INIT }
+
+static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
+  return TryAcquireSRWLockExclusive(&lock->mutex);
+}
+static inline void mi_lock_acquire(mi_lock_t* lock) {
+  AcquireSRWLockExclusive(&lock->mutex);
+}
+static inline void mi_lock_release(mi_lock_t* lock) {
+  ReleaseSRWLockExclusive(&lock->mutex);
+}
+static inline void mi_lock_init(mi_lock_t* lock) {
+  InitializeSRWLock(&lock->mutex);
+}
+static inline void mi_lock_done(mi_lock_t* lock) {
+  (void)(lock);
+}
+
+#elif defined(MI_USE_PTHREADS)
+
+#include <string.h> // memcpy
+void _mi_error_message(int err, const char* fmt, ...);
+
+typedef struct mi_lock_s {
+  pthread_mutex_t mutex;
+} mi_lock_t;
+
+#define MI_LOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
+
+static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
+  return (pthread_mutex_trylock(&lock->mutex) == 0);
+}
+static inline void mi_lock_acquire(mi_lock_t* lock) {
+  const int err = pthread_mutex_lock(&lock->mutex);
+  if (err != 0) {
+    _mi_error_message(err, "internal error: lock cannot be acquired (err %i)\n", err);
+  }
+}
+static inline void mi_lock_release(mi_lock_t* lock) {
+  pthread_mutex_unlock(&lock->mutex);
+}
+static inline void mi_lock_init(mi_lock_t* lock) {
+  if(lock==NULL) return;
+  // use this instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)
+  const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+  memcpy(&lock->mutex,&mutex,sizeof(mutex));
+}
+static inline void mi_lock_done(mi_lock_t* lock) {
+  pthread_mutex_destroy(&lock->mutex);
+}
+
+#elif defined(__cplusplus)
+
+#include <thread>
+#include <mutex>
+#include <new>
+
+typedef struct mi_lock_s {
+  std::mutex mutex;
+} mi_lock_t;
+
+#define MI_LOCK_INITIALIZER   { }
+
+static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
+  return lock->mutex.try_lock();
+}
+static inline void mi_lock_acquire(mi_lock_t* lock) {
+  lock->mutex.lock();
+}
+static inline void mi_lock_release(mi_lock_t* lock) {
+  lock->mutex.unlock();
+}
+static inline void mi_lock_init(mi_lock_t* lock) {
+  new(&lock->mutex) std::mutex();
+}
+static inline void mi_lock_done(mi_lock_t* lock) {
+  (void)(lock);
+}
+
+#else
+
+// fall back to poor man's locks.
+// this should only be the case in a single-threaded environment (like __wasi__)
+#include <errno.h>
+#ifndef EFAULT
+#define EFAULT (14)
+#endif
+void _mi_error_message(int err, const char* fmt, ...);
+void _mi_prim_thread_yield(void);
+
+typedef struct mi_lock_s {
+  _Atomic(uintptr_t) mutex;
+} mi_lock_t;
+
+#define MI_LOCK_INITIALIZER  { MI_ATOMIC_VAR_INIT(0) }
+
+static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
+  uintptr_t expected = 0;
+  return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);
+}
+static inline void mi_lock_acquire(mi_lock_t* lock) {
+  for (int i = 0; i < 10000; i++) {  // for at most 10000 tries?
+    if (mi_lock_try_acquire(lock)) return;
+    _mi_prim_thread_yield();
+  }
+  _mi_error_message(EFAULT, "internal error: lock cannot be acquired (due to lack of native lock primitives)\n");
+}
+static inline void mi_lock_release(mi_lock_t* lock) {
+  mi_atomic_store_release(&lock->mutex, (uintptr_t)0);
+}
+static inline void mi_lock_init(mi_lock_t* lock) {
+  mi_lock_release(lock);
+}
+static inline void mi_lock_done(mi_lock_t* lock) {
+  (void)(lock);
+}
+
+#endif
+
+
+typedef struct mi_atomic_once_s {
+  _Atomic(uintptr_t) tid;
+  mi_lock_t          lock;
+} mi_atomic_once_t;
+
+// Returns `true` only on the first invocation, signifying we can execute an action once.
+// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
+// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
+bool _mi_atomic_once_enter(mi_atomic_once_t* once);        // defined in `libc.c`
+void _mi_atomic_once_release(mi_atomic_once_t* once);      // defined in `libc.c`
+
+#define mi_atomic_do_once  \
+  static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \
+  for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false))
+
+
+#endif // __MIMALLOC_ATOMIC_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/bits.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/bits.h
new file mode 100644
index 00000000..61787728
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/bits.h
@@ -0,0 +1,340 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+/* ----------------------------------------------------------------------------
+  Bit operation, and platform dependent definition (MI_INTPTR_SIZE etc)
+---------------------------------------------------------------------------- */
+
+#pragma once
+#ifndef MI_BITS_H
+#define MI_BITS_H
+
+#include <stddef.h>   // size_t
+#include <stdint.h>   // int64_t etc
+#include <stdbool.h>  // bool
+
+// ------------------------------------------------------
+// Size of a pointer.
+// We assume that `sizeof(void*)==sizeof(intptr_t)`
+// and it holds for all platforms we know of.
+//
+// However, the C standard only requires that:
+//  p == (void*)((intptr_t)p))
+// but we also need:
+//  i == (intptr_t)((void*)i)
+// or otherwise one might define an intptr_t type that is larger than a pointer...
+// ------------------------------------------------------
+
+#if INTPTR_MAX > INT64_MAX
+# define MI_INTPTR_SHIFT (4)  // assume 128-bit  (as on arm CHERI for example)
+#elif INTPTR_MAX == INT64_MAX
+# define MI_INTPTR_SHIFT (3)
+#elif INTPTR_MAX == INT32_MAX
+# define MI_INTPTR_SHIFT (2)
+#else
+#error platform pointers must be 32, 64, or 128 bits
+#endif
+
+#if (INTPTR_MAX) > LONG_MAX
+# define MI_PU(x)  x##ULL
+#else
+# define MI_PU(x)  x##UL
+#endif
+
+#if SIZE_MAX == UINT64_MAX
+# define MI_SIZE_SHIFT (3)
+typedef int64_t  mi_ssize_t;
+#elif SIZE_MAX == UINT32_MAX
+# define MI_SIZE_SHIFT (2)
+typedef int32_t  mi_ssize_t;
+#else
+#error platform objects must be 32 or 64 bits in size
+#endif
+
+#if (SIZE_MAX/2) > LONG_MAX
+# define MI_ZU(x)  x##ULL
+#else
+# define MI_ZU(x)  x##UL
+#endif
+
+#define MI_INTPTR_SIZE  (1<<MI_INTPTR_SHIFT)
+#define MI_INTPTR_BITS  (MI_INTPTR_SIZE*8)
+
+#define MI_SIZE_SIZE  (1<<MI_SIZE_SHIFT)
+#define MI_SIZE_BITS  (MI_SIZE_SIZE*8)
+
+#define MI_KiB     (MI_ZU(1024))
+#define MI_MiB     (MI_KiB*MI_KiB)
+#define MI_GiB     (MI_MiB*MI_KiB)
+
+
+/* --------------------------------------------------------------------------------
+  Architecture
+-------------------------------------------------------------------------------- */
+
+#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_HYBRID_X86_ARM64) || defined(_M_ARM64EC)  // consider arm64ec as arm64
+#define MI_ARCH_ARM64     1
+#elif defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64)
+#define MI_ARCH_X64       1
+#elif defined(__i386__) || defined(__i386) || defined(_M_IX86) || defined(_X86_) || defined(__X86__)
+#define MI_ARCH_X86       1
+#elif defined(__arm__) || defined(_ARM) || defined(_M_ARM)  || defined(_M_ARMT) || defined(__arm)
+#define MI_ARCH_ARM32     1
+#elif defined(__riscv) || defined(_M_RISCV)
+#define MI_ARCH_RISCV     1
+#if (LONG_MAX == INT32_MAX)
+#define MI_ARCH_RISCV32   1
+#else
+#define MI_ARCH_RISCV64   1
+#endif
+#endif
+
+#if MI_ARCH_X64 && defined(__AVX2__)
+#include <immintrin.h>
+#elif MI_ARCH_ARM64 && MI_OPT_SIMD
+#include <arm_neon.h>
+#endif
+#if defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
+#include <intrin.h>
+#endif
+
+#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // msvc
+#define __BMI2__  1
+#endif
+#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc
+#define __BMI1__  1
+#endif
+#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc
+#define __LZCNT__  1
+#endif
+
+// Define big endian if needed
+// #define MI_BIG_ENDIAN  1
+
+// maximum virtual address bits in a user-space pointer
+#if MI_DEFAULT_VIRTUAL_ADDRESS_BITS > 0 
+#define MI_MAX_VABITS     MI_DEFAULT_VIRTUAL_ADDRESS_BITS
+#elif   MI_ARCH_X64
+#define MI_MAX_VABITS     (47)
+#elif MI_INTPTR_SIZE > 4
+#define MI_MAX_VABITS     (48)
+#else
+#define MI_MAX_VABITS     (32)
+#endif
+
+// use a flat page-map or a 2-level one
+#ifndef MI_PAGE_MAP_FLAT
+#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED
+#define MI_PAGE_MAP_FLAT  1
+#else
+#define MI_PAGE_MAP_FLAT  0
+#endif
+#endif
+
+
+/* --------------------------------------------------------------------------------
+  Builtin's
+-------------------------------------------------------------------------------- */
+
+#ifndef __has_builtin
+#define __has_builtin(x)  0
+#endif
+
+#define mi_builtin(name)        __builtin_##name
+#define mi_has_builtin(name)    __has_builtin(__builtin_##name)
+
+#if (LONG_MAX == INT32_MAX)
+#define mi_builtin32(name)       mi_builtin(name##l)
+#define mi_has_builtin32(name)   mi_has_builtin(name##l)
+#else
+#define mi_builtin32(name)       mi_builtin(name)
+#define mi_has_builtin32(name)   mi_has_builtin(name)
+#endif
+#if (LONG_MAX == INT64_MAX)
+#define mi_builtin64(name)       mi_builtin(name##l)
+#define mi_has_builtin64(name)   mi_has_builtin(name##l)
+#else
+#define mi_builtin64(name)       mi_builtin(name##ll)
+#define mi_has_builtin64(name)   mi_has_builtin(name##ll)
+#endif
+
+#if (MI_SIZE_BITS == 32)
+#define mi_builtinz(name)        mi_builtin32(name)
+#define mi_has_builtinz(name)    mi_has_builtin32(name)
+#define mi_msc_builtinz(name)    name
+#elif (MI_SIZE_BITS == 64)
+#define mi_builtinz(name)        mi_builtin64(name)
+#define mi_has_builtinz(name)    mi_has_builtin64(name)
+#define mi_msc_builtinz(name)    name##64
+#endif
+
+/* --------------------------------------------------------------------------------
+  Popcount and count trailing/leading zero's
+-------------------------------------------------------------------------------- */
+
+size_t _mi_popcount_generic(size_t x);
+
+static inline size_t mi_popcount(size_t x) {
+  #if mi_has_builtinz(popcount)
+    return mi_builtinz(popcount)(x);
+  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
+    return mi_msc_builtinz(__popcnt)(x);
+  #elif MI_ARCH_X64 && defined(__BMI1__)
+    return (size_t)_mm_popcnt_u64(x);
+  #else
+    #define MI_HAS_FAST_POPCOUNT  0
+    return (x<=1 ? x : _mi_popcount_generic(x));
+  #endif
+}
+
+#ifndef MI_HAS_FAST_POPCOUNT
+#define MI_HAS_FAST_POPCOUNT 1
+#endif
+
+
+
+size_t _mi_clz_generic(size_t x);
+size_t _mi_ctz_generic(size_t x);
+
+static inline size_t mi_ctz(size_t x) {
+  #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) // on x64 tzcnt is defined for 0
+    size_t r;
+    __asm ("tzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
+    return r;
+  #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__) 
+    return _tzcnt_u64(x);
+  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
+    unsigned long idx;
+    return (mi_msc_builtinz(_BitScanForward)(&idx, x) ? (size_t)idx : MI_SIZE_BITS);
+  #elif mi_has_builtinz(ctz)
+    return (x!=0 ? (size_t)mi_builtinz(ctz)(x) : MI_SIZE_BITS);
+  #elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)
+    size_t r = MI_SIZE_BITS;  // bsf leaves destination unmodified if the argument is 0 (see <https://github.com/llvm/llvm-project/pull/102885>)
+    __asm ("bsf\t%1, %0" : "+r"(r) : "r"(x) : "cc");
+    return r;
+  #elif MI_HAS_FAST_POPCOUNT
+    return (x!=0 ? (mi_popcount(x^(x-1))-1) : MI_SIZE_BITS);
+  #else
+    #define MI_HAS_FAST_BITSCAN  0
+    return (x!=0 ? _mi_ctz_generic(x) : MI_SIZE_BITS);
+  #endif
+}
+
+static inline size_t mi_clz(size_t x) {
+  #if defined(__GNUC__) && MI_ARCH_X64 && defined(__LZCNT__) // on x64 lzcnt is defined for 0
+    size_t r;
+    __asm ("lzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
+    return r;
+  #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__) 
+    return _lzcnt_u64(x);
+  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
+    unsigned long idx;
+    return (mi_msc_builtinz(_BitScanReverse)(&idx, x) ? MI_SIZE_BITS - 1 - (size_t)idx : MI_SIZE_BITS);
+  #elif mi_has_builtinz(clz)
+    return (x!=0 ? (size_t)mi_builtinz(clz)(x) : MI_SIZE_BITS);
+  #elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)
+    if (x==0) return MI_SIZE_BITS;
+    size_t r;
+    __asm ("bsr\t%1, %0" : "=r"(r) : "r"(x) : "cc");
+    return (MI_SIZE_BITS - 1 - r);
+  #else
+    #define MI_HAS_FAST_BITSCAN  0
+    return (x!=0 ? _mi_clz_generic(x) : MI_SIZE_BITS);
+  #endif
+}
+
+#ifndef MI_HAS_FAST_BITSCAN
+#define MI_HAS_FAST_BITSCAN 1
+#endif
+
+/* --------------------------------------------------------------------------------
+  find trailing/leading zero  (bit scan forward/reverse)
+-------------------------------------------------------------------------------- */
+
+// Bit scan forward: find the least significant bit that is set (i.e. count trailing zero's)
+// return false if `x==0` (with `*idx` undefined) and true otherwise,
+// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
+static inline bool mi_bsf(size_t x, size_t* idx) {
+  #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) && (!defined(__clang_major__) || __clang_major__ >= 9)
+    // on x64 the carry flag is set on zero which gives better codegen
+    bool is_zero;
+    __asm ( "tzcnt\t%2, %1" : "=@ccc"(is_zero), "=r"(*idx) : "r"(x) : "cc" );
+    return !is_zero;
+  #elif 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
+    unsigned long i;
+    return (mi_msc_builtinz(_BitScanForward)(&i, x) ? (*idx = (size_t)i, true) : false);
+  #else
+    return (x!=0 ? (*idx = mi_ctz(x), true) : false);
+  #endif
+}
+
+// Bit scan reverse: find the most significant bit that is set
+// return false if `x==0` (with `*idx` undefined) and true otherwise,
+// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
+static inline bool mi_bsr(size_t x, size_t* idx) {
+  #if 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
+    unsigned long i;
+    return (mi_msc_builtinz(_BitScanReverse)(&i, x) ? (*idx = (size_t)i, true) : false);
+  #else
+    return (x!=0 ? (*idx = MI_SIZE_BITS - 1 - mi_clz(x), true) : false);
+  #endif
+}
+
+
+/* --------------------------------------------------------------------------------
+  rotate
+-------------------------------------------------------------------------------- */
+
+static inline size_t mi_rotr(size_t x, size_t r) {
+  #if (mi_has_builtin(rotateright64) && MI_SIZE_BITS==64)
+    return mi_builtin(rotateright64)(x,r);
+  #elif (mi_has_builtin(rotateright32) && MI_SIZE_BITS==32)
+    return mi_builtin(rotateright32)(x,r);
+  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)
+    return _rotr64(x, (int)r);
+  #elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)
+    return _lrotr(x,(int)r);
+  #else
+    // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
+    // avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
+    const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
+    return ((x >> rshift) | (x << ((-rshift) & (MI_SIZE_BITS-1))));
+  #endif
+}
+
+static inline size_t mi_rotl(size_t x, size_t r) {
+  #if (mi_has_builtin(rotateleft64) && MI_SIZE_BITS==64)
+    return mi_builtin(rotateleft64)(x,r);
+  #elif (mi_has_builtin(rotateleft32) && MI_SIZE_BITS==32)
+    return mi_builtin(rotateleft32)(x,r);
+  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)
+    return _rotl64(x, (int)r);
+  #elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)
+    return _lrotl(x, (int)r);
+  #else
+    // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
+    // avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
+    const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
+    return ((x << rshift) | (x >> ((-rshift) & (MI_SIZE_BITS-1))));
+  #endif
+}
+
+static inline uint32_t mi_rotl32(uint32_t x, uint32_t r) {
+  #if mi_has_builtin(rotateleft32)
+    return mi_builtin(rotateleft32)(x,r);
+  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
+    return _lrotl(x, (int)r);
+  #else
+    // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
+    // avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
+    const unsigned int rshift = (unsigned int)(r) & 31;
+    return ((x << rshift) | (x >> ((-rshift) & 31)));
+  #endif
+}
+
+
+#endif // MI_BITS_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/internal.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/internal.h
new file mode 100644
index 00000000..febf2559
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/internal.h
@@ -0,0 +1,1282 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MI_INTERNAL_H
+#define MI_INTERNAL_H
+
+// --------------------------------------------------------------------------
+// This file contains the internal API's of mimalloc and various utility
+// functions and macros.
+// --------------------------------------------------------------------------
+
+#include "types.h"
+#include "track.h"
+#include "bits.h"
+
+
+// --------------------------------------------------------------------------
+// Compiler defines
+// --------------------------------------------------------------------------
+
+#if (MI_DEBUG>0)
+#define mi_trace_message(...)  _mi_trace_message(__VA_ARGS__)
+#else
+#define mi_trace_message(...)
+#endif
+
+#define mi_decl_cache_align     mi_decl_align(64)
+
+#if defined(_MSC_VER)
+#pragma warning(disable:4127)   // suppress constant conditional warning (due to MI_SECURE paths)
+#pragma warning(disable:26812)  // unscoped enum warning
+#define mi_decl_forceinline     __forceinline
+#define mi_decl_noinline        __declspec(noinline)
+#define mi_decl_thread          __declspec(thread)
+#define mi_decl_noreturn        __declspec(noreturn)
+#define mi_decl_weak
+#define mi_decl_hidden
+#define mi_decl_cold
+#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
+#if !MI_TRACK_ASAN
+#define mi_decl_forceinline     __attribute__((always_inline)) inline
+#else
+#define mi_decl_forceinline     inline
+#endif
+#define mi_decl_noinline        __attribute__((noinline))
+#define mi_decl_thread          __thread
+#define mi_decl_noreturn        __attribute__((noreturn))
+#define mi_decl_weak            __attribute__((weak))
+#define mi_decl_hidden          __attribute__((visibility("hidden")))
+#if (__GNUC__ >= 4) || defined(__clang__)
+#define mi_decl_cold            __attribute__((cold))
+#else
+#define mi_decl_cold
+#endif
+#elif __cplusplus >= 201103L    // c++11
+#define mi_decl_forceinline     inline
+#define mi_decl_noinline
+#define mi_decl_thread          thread_local
+#define mi_decl_noreturn        [[noreturn]]
+#define mi_decl_weak
+#define mi_decl_hidden
+#define mi_decl_cold
+#else
+#define mi_decl_forceinline     inline
+#define mi_decl_noinline
+#define mi_decl_thread          __thread        // hope for the best :-)
+#define mi_decl_noreturn
+#define mi_decl_weak
+#define mi_decl_hidden
+#define mi_decl_cold
+#endif
+
+#if defined(__GNUC__) || defined(__clang__)
+#define mi_unlikely(x)     (__builtin_expect(!!(x),false))
+#define mi_likely(x)       (__builtin_expect(!!(x),true))
+#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)
+#define mi_unlikely(x)     (x) [[unlikely]]
+#define mi_likely(x)       (x) [[likely]]
+#else
+#define mi_unlikely(x)     (x)
+#define mi_likely(x)       (x)
+#endif
+
+#ifndef __has_builtin
+#define __has_builtin(x)    0
+#endif
+
+#if defined(__cplusplus)
+#define mi_decl_externc     extern "C"
+#else
+#define mi_decl_externc
+#endif
+
+#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc
+#define mi_decl_maybe_unused    __attribute__((unused))
+#elif __cplusplus >= 201703L    // c++17
+#define mi_decl_maybe_unused    [[maybe_unused]]
+#else
+#define mi_decl_maybe_unused
+#endif
+
+#if defined(__cplusplus)
+#define mi_decl_externc         extern "C"
+#else
+#define mi_decl_externc
+#endif
+
+
+#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
+#define __wasi__
+#endif
+
+
+// --------------------------------------------------------------------------
+// Internal functions
+// --------------------------------------------------------------------------
+
+
+// "libc.c"
+#include <stdarg.h>
+int           _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args);
+int           _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);
+char          _mi_toupper(char c);
+int           _mi_strnicmp(const char* s, const char* t, size_t n);
+void          _mi_strlcpy(char* dest, const char* src, size_t dest_size);
+void          _mi_strlcat(char* dest, const char* src, size_t dest_size);
+size_t        _mi_strlen(const char* s);
+size_t        _mi_strnlen(const char* s, size_t max_len);
+char*         _mi_strnstr(char* s, size_t max_len, const char* pat);
+bool          _mi_streq(const char* s, const char* t);
+bool          _mi_getenv(const char* name, char* result, size_t result_size);
+
+// "options.c"
+void          _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
+void          _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
+void          _mi_raw_message(const char* fmt, ...);
+void          _mi_message(const char* fmt, ...);
+void          _mi_warning_message(const char* fmt, ...);
+void          _mi_verbose_message(const char* fmt, ...);
+void          _mi_trace_message(const char* fmt, ...);
+void          _mi_options_init(void);
+void          _mi_options_post_init(void);
+long          _mi_option_get_fast(mi_option_t option);
+void          _mi_error_message(int err, const char* fmt, ...);
+
+// random.c
+void          _mi_random_init(mi_random_ctx_t* ctx);
+void          _mi_random_init_weak(mi_random_ctx_t* ctx);
+void          _mi_random_reinit_if_weak(mi_random_ctx_t * ctx);
+void          _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);
+uintptr_t     _mi_random_next(mi_random_ctx_t* ctx);
+uintptr_t     _mi_theap_random_next(mi_theap_t* theap);
+uintptr_t     _mi_os_random_weak(uintptr_t extra_seed);
+static inline uintptr_t _mi_random_shuffle(uintptr_t x);
+
+// init.c
+extern mi_decl_hidden mi_decl_cache_align const mi_page_t  _mi_page_empty;
+void          _mi_auto_process_init(void);
+void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept;
+bool          _mi_is_redirected(void);
+bool          _mi_allocator_init(const char** message);
+void          _mi_allocator_done(void);
+bool          _mi_is_main_thread(void);
+bool          _mi_preloading(void);           // true while the C runtime is not initialized yet
+void          _mi_thread_done(mi_theap_t* theap);
+
+mi_subproc_t* _mi_subproc(void);
+mi_subproc_t* _mi_subproc_main(void);
+mi_heap_t*    _mi_subproc_heap_main(mi_subproc_t* subproc);
+mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);
+
+mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
+size_t        _mi_thread_seq_id(void) mi_attr_noexcept;
+bool          _mi_is_heap_main(const mi_heap_t* heap);
+bool          _mi_is_theap_main(const mi_theap_t* theap);
+void          _mi_theap_guarded_init(mi_theap_t* theap);
+void          _mi_theap_options_init(mi_theap_t* theap);
+mi_theap_t*   _mi_theap_default_safe(void);             // ensure the returned theap is initialized
+mi_theap_t*   _mi_theap_main_safe(void);
+   
+// os.c
+void          _mi_os_init(void);                                            // called from process init
+void*         _mi_os_alloc(size_t size, mi_memid_t* memid);
+void*         _mi_os_zalloc(size_t size, mi_memid_t* memid);
+void          _mi_os_free(void* p, size_t size, mi_memid_t memid);
+void          _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc );
+
+size_t        _mi_os_page_size(void);
+size_t        _mi_os_guard_page_size(void);
+size_t        _mi_os_good_alloc_size(size_t size);
+bool          _mi_os_has_overcommit(void);
+bool          _mi_os_has_virtual_reserve(void);
+size_t        _mi_os_virtual_address_bits(void);
+size_t        _mi_os_minimal_purge_size(void);
+
+bool          _mi_os_reset(void* addr, size_t size);
+bool          _mi_os_decommit(void* addr, size_t size);
+void          _mi_os_reuse(void* p, size_t size);
+mi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero);
+mi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size);
+mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size);
+bool          _mi_os_unprotect(void* addr, size_t size);
+bool          _mi_os_purge(void* p, size_t size);
+bool          _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg);
+
+size_t        _mi_os_secure_guard_page_size(void);
+bool          _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid);
+bool          _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid);
+bool          _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid);
+bool          _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid);
+
+int           _mi_os_numa_node(void);
+int           _mi_os_numa_node_count(void);
+
+void*         _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);
+void*         _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);
+
+void*         _mi_os_get_aligned_hint(size_t try_alignment, size_t size);
+bool          _mi_os_canuse_large_page(size_t size, size_t alignment);
+size_t        _mi_os_large_page_size(void);
+void*         _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);
+
+// threadlocal.c
+
+mi_thread_local_t _mi_thread_local_create(void);
+void          _mi_thread_local_free( mi_thread_local_t key );
+bool          _mi_thread_local_set(  mi_thread_local_t key, void* val );
+void*         _mi_thread_local_get(  mi_thread_local_t key );
+void          _mi_thread_locals_init(void);
+void          _mi_thread_locals_done(void);
+void          _mi_thread_locals_thread_done(void);
+
+// arena.c
+mi_arena_id_t _mi_arena_id_none(void);
+mi_arena_t*   _mi_arena_from_id(mi_arena_id_t id);
+bool          _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena);
+
+void*         _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);
+void*         _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);
+void          _mi_arenas_free(void* p, size_t size, mi_memid_t memid);
+bool          _mi_arenas_contain(const void* p);
+void          _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld);
+void          _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc);
+
+mi_page_t*    _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment);
+void          _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);
+void          _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap);
+void          _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);
+bool          _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page);
+
+// arena-meta.c
+void*         _mi_meta_zalloc( size_t size, mi_memid_t* memid );
+void          _mi_meta_free(void* p, size_t size, mi_memid_t memid);
+bool          _mi_meta_is_meta_page(void* p);
+
+// "page-map.c"
+bool          _mi_page_map_init(void);
+mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page);
+void          _mi_page_map_unregister(mi_page_t* page);
+void          _mi_page_map_unregister_range(void* start, size_t size);
+mi_page_t*    _mi_safe_ptr_page(const void* p);
+void          _mi_page_map_unsafe_destroy(mi_subproc_t* subproc);
+
+// "page.c"
+void*         _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable)  mi_attr_noexcept mi_attr_malloc;
+
+void          _mi_page_retire(mi_page_t* page) mi_attr_noexcept;       // free the page if there are no other pages with many free blocks
+void          _mi_page_unfull(mi_page_t* page);
+void          _mi_page_free(mi_page_t* page, mi_page_queue_t* pq);     // free the page
+void          _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq);  // abandon the page, to be picked up by another thread...
+
+size_t        _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append);
+void          _mi_deferred_free(mi_theap_t* theap, bool force);
+
+void          _mi_page_free_collect(mi_page_t* page, bool force);
+void          _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head);
+mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page);
+bool          _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq);
+
+size_t        _mi_page_stats_bin(const mi_page_t* page); // for stats
+size_t        _mi_bin_size(size_t bin);                  // for stats
+size_t        _mi_bin(size_t size);                      // for stats
+
+// "theap.c"
+mi_theap_t*   _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld);
+void          _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock);
+void          _mi_theap_default_set(mi_theap_t* theap);
+void          _mi_theap_cached_set(mi_theap_t* theap);
+void          _mi_theap_collect_retired(mi_theap_t* theap, bool force);
+void          _mi_theap_collect_abandon(mi_theap_t* theap);
+bool          _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);
+void          _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page);
+bool          _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock);
+void          _mi_theap_incref(mi_theap_t* theap);
+void          _mi_theap_decref(mi_theap_t* theap);
+
+// "heap.c"
+void          _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);
+mi_decl_cold  mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap);  // get (and possible create) the theap belonging to a heap
+mi_decl_cold  mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap);     // get the theap for a heap without initializing (and return NULL in that case)
+void          _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to);  // in "arena.c"
+void          _mi_heap_destroy_pages(mi_heap_t* heap_from);                   // in "arena.c"
+void          _mi_heap_force_destroy(mi_heap_t* heap);                        // allow destroying the main heap
+
+// "stats.c"
+void          _mi_stats_init(void);
+void          _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from);
+
+mi_msecs_t    _mi_clock_now(void);
+mi_msecs_t    _mi_clock_end(mi_msecs_t start);
+mi_msecs_t    _mi_clock_start(void);
+
+// "alloc.c"
+void*         _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept;                  // called from `_mi_theap_malloc_aligned`
+void*         _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept;
+void*         _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept;     // called from `_mi_theap_malloc_aligned`
+void*         _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept;
+mi_block_t*   _mi_page_ptr_unalign(const mi_page_t* page, const void* p);
+void          _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size);
+
+#if MI_DEBUG>1
+bool          _mi_page_is_valid(mi_page_t* page);
+#endif
+
+
+// ------------------------------------------------------
+// Assertions
+// ------------------------------------------------------
+
+#if (MI_DEBUG)
+// use our own assertion to print without memory allocation
+mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept;
+#define mi_assert(expr)     ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__))
+#else
+#define mi_assert(x)
+#endif
+
+#if (MI_DEBUG>1)
+#define mi_assert_internal    mi_assert
+#else
+#define mi_assert_internal(x)
+#endif
+
+#if (MI_DEBUG>2)
+#define mi_assert_expensive   mi_assert
+#else
+#define mi_assert_expensive(x)
+#endif
+
+
+/* -----------------------------------------------------------
+  Statistics (in `stats.c`)
+----------------------------------------------------------- */
+
+// add to stat keeping track of the peak
+void __mi_stat_increase(mi_stat_count_t* stat, size_t amount);
+void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount);
+void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount);
+void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount);
+
+// adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total)
+void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount);
+void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount);
+void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount);
+void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount);
+
+// counters can just be increased
+void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
+void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);
+
+#define mi_heap_stat_counter_increase(heap,stat,amount)         __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount)
+#define mi_heap_stat_increase(heap,stat,amount)                 __mi_stat_increase_mt( &(heap)->stats.stat, amount)
+#define mi_heap_stat_decrease(heap,stat,amount)                 __mi_stat_decrease_mt( &(heap)->stats.stat, amount)
+#define mi_heap_stat_adjust_increase(heap,stat,amnt)            __mi_stat_adjust_increase_mt( &(heap)->stats.stat, amnt)
+#define mi_heap_stat_adjust_decrease(heap,stat,amnt)            __mi_stat_adjust_decrease_mt( &(heap)->stats.stat, amnt)
+
+#define mi_subproc_stat_counter_increase(subproc,stat,amount)   __mi_stat_counter_increase_mt( &(subproc)->stats.stat, amount)
+#define mi_subproc_stat_increase(subproc,stat,amount)           __mi_stat_increase_mt( &(subproc)->stats.stat, amount)
+#define mi_subproc_stat_decrease(subproc,stat,amount)           __mi_stat_decrease_mt( &(subproc)->stats.stat, amount)
+#define mi_subproc_stat_adjust_increase(subproc,stat,amount)    __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amount)
+#define mi_subproc_stat_adjust_decrease(subproc,stat,amount)    __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount)
+
+#define mi_os_stat_counter_increase(stat,amount)                mi_subproc_stat_counter_increase(_mi_subproc(),stat,amount)
+#define mi_os_stat_increase(stat,amount)                        mi_subproc_stat_increase(_mi_subproc(),stat,amount)
+#define mi_os_stat_decrease(stat,amount)                        mi_subproc_stat_decrease(_mi_subproc(),stat,amount)
+
+#define mi_theap_stat_counter_increase(theap,stat,amount)       __mi_stat_counter_increase( &(theap)->stats.stat, amount)
+#define mi_theap_stat_increase(theap,stat,amount)               __mi_stat_increase( &(theap)->stats.stat, amount)
+#define mi_theap_stat_decrease(theap,stat,amount)               __mi_stat_decrease( &(theap)->stats.stat, amount)
+#define mi_theap_stat_adjust_increase(theap,stat,amnt)          __mi_stat_adjust_increase( &(theap)->stats.stat, amnt)
+#define mi_theap_stat_adjust_decrease(theap,stat,amnt)          __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt)
+
+
+/* -----------------------------------------------------------
+  Options (exposed for the debugger)
+----------------------------------------------------------- */
+typedef enum mi_option_init_e {
+  MI_OPTION_UNINIT,       // not yet initialized
+  MI_OPTION_DEFAULTED,    // not found in the environment, use default value
+  MI_OPTION_INITIALIZED   // found in environment or set explicitly
+} mi_option_init_t;
+
+typedef struct mi_option_desc_s {
+  long              value;  // the value
+  mi_option_init_t  init;   // is it initialized yet? (from the environment)
+  mi_option_t       option; // for debugging: the option index should match the option
+  const char*       name;   // option name without `mimalloc_` prefix
+  const char*       legacy_name; // potential legacy option name
+} mi_option_desc_t;
+
+
+
+/* -----------------------------------------------------------
+  Inlined definitions
+----------------------------------------------------------- */
+#define MI_UNUSED(x)     (void)(x)
+#if (MI_DEBUG>1)
+#define MI_UNUSED_RELEASE(x)
+#else
+#define MI_UNUSED_RELEASE(x)  MI_UNUSED(x)
+#endif
+
+#define MI_INIT4(x)   x(),x(),x(),x()
+#define MI_INIT8(x)   MI_INIT4(x),MI_INIT4(x)
+#define MI_INIT16(x)  MI_INIT8(x),MI_INIT8(x)
+#define MI_INIT32(x)  MI_INIT16(x),MI_INIT16(x)
+#define MI_INIT64(x)  MI_INIT32(x),MI_INIT32(x)
+#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x)
+#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x)
+
+#define MI_INIT74(x)  MI_INIT64(x),MI_INIT8(x),x(),x()
+#define MI_INIT5(x)   MI_INIT4(x),x()
+#define MI_INIT6(x)   MI_INIT4(x),x(),x()
+
+#include <string.h>
+// initialize a local variable to zero; use memset as compilers optimize constant sized memset's
+#define _mi_memzero_var(x)  memset(&x,0,sizeof(x))
+
+// Is `x` a power of two? (0 is considered a power of two)
+static inline bool _mi_is_power_of_two(uintptr_t x) {
+  return ((x & (x - 1)) == 0);
+}
+
+// Is a pointer aligned?
+static inline bool _mi_is_aligned(const void* p, size_t alignment) {
+  return (alignment==0 || ((uintptr_t)p % alignment) == 0);
+}
+
+// Align upwards
+static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
+  mi_assert_internal(alignment != 0);
+  uintptr_t mask = alignment - 1;
+  if ((alignment & mask) == 0) {  // power of two?
+    return ((sz + mask) & ~mask);
+  }
+  else {
+    return (((sz + mask)/alignment)*alignment);
+  }
+}
+
+// Align a pointer upwards
+static inline void* _mi_align_up_ptr(const void* p, size_t alignment) {
+  return (void*)_mi_align_up((uintptr_t)p, alignment);
+}
+
+static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
+  mi_assert_internal(alignment != 0);
+  uintptr_t mask = alignment - 1;
+  if ((alignment & mask) == 0) { // power of two?
+    return (sz & ~mask);
+  }
+  else {
+    return ((sz / alignment) * alignment);
+  }
+}
+
+// align a pointer downwards
+static inline void* _mi_align_down_ptr(const void* p, size_t alignment) {
+  return (void*)_mi_align_down((uintptr_t)p, alignment);
+}
+
+// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`.
+static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) {
+  mi_assert_internal(divider != 0);
+  return (divider == 0 ? size : ((size + divider - 1) / divider));
+}
+
+
+// clamp an integer
+static inline size_t _mi_clamp(size_t sz, size_t min, size_t max) {
+  if (sz < min) return min;
+  else if (sz > max) return max;
+  else return sz;
+}
+
+// Is memory zero initialized?
+static inline bool mi_mem_is_zero(const void* p, size_t size) {
+  for (size_t i = 0; i < size; i++) {
+    if (((uint8_t*)p)[i] != 0) return false;
+  }
+  return true;
+}
+
+// Align a byte size to a size in _machine words_,
+// i.e. byte size == `wsize*sizeof(void*)`.
+static inline size_t _mi_wsize_from_size(size_t size) {
+  mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t));
+  return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);
+}
+
+// Overflow detecting multiply
+#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5))
+#include <limits.h>      // UINT_MAX, ULONG_MAX
+#if defined(_CLOCK_T)    // for Illumos
+#undef _CLOCK_T
+#endif
+static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
+  #if (SIZE_MAX == ULONG_MAX)
+    return __builtin_umull_overflow(count, size, (unsigned long *)total);
+  #elif (SIZE_MAX == UINT_MAX)
+    return __builtin_umul_overflow(count, size, (unsigned int *)total);
+  #else
+    return __builtin_umulll_overflow(count, size, (unsigned long long *)total);
+  #endif
+}
+#else /* __builtin_umul_overflow is unavailable */
+static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
+  *total = count*size;
+  if mi_likely(((size|count)>>(4*MI_SIZE_SIZE))==0) {  // did size and count fit both in the lower half bits of a size_t?
+    return false;
+  }
+  else {
+    return (size!=0 && (SIZE_MAX / size) < count);
+  }
+}
+#endif
+
+// Safe multiply `count*size` into `total`; return `true` on overflow.
+static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) {
+  if (count==1) {  // quick check for the case where count is one (common for C++ allocators)
+    *total = size;
+    return false;
+  }
+  else if mi_likely(!mi_mul_overflow(count, size, total)) {
+    return false;
+  }
+  else {
+    #if MI_DEBUG > 0
+    _mi_error_message(EOVERFLOW, "allocation request is too large (%zu * %zu bytes)\n", count, size);
+    #endif
+    *total = SIZE_MAX;
+    return true;
+  }
+}
+
+
+/*----------------------------------------------------------------------------------------
+  Heap functions
+------------------------------------------------------------------------------------------- */
+
+extern mi_decl_hidden const mi_theap_t _mi_theap_empty;       // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_THREAD_LOCAL)
+extern mi_decl_hidden const mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated
+
+
+static inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) {
+  return mi_atomic_load_ptr_acquire(mi_heap_t,&theap->heap);
+}
+
+static inline bool mi_theap_is_initialized(const mi_theap_t* theap) {
+  return (theap != NULL && _mi_theap_heap(theap) != NULL);
+}
+
+static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) {
+  mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));
+  const size_t idx = _mi_wsize_from_size(size);
+  mi_assert_internal(idx < MI_PAGES_DIRECT);
+  return theap->pages_free_direct[idx];
+}
+
+
+//static inline uintptr_t _mi_ptr_cookie(const void* p) {
+//  extern mi_theap_t _mi_theap_main;
+//  mi_assert_internal(_mi_theap_main.cookie != 0);
+//  return ((uintptr_t)p ^ _mi_theap_main.cookie);
+//}
+
+
+/* -----------------------------------------------------------
+  The page map maps addresses to `mi_page_t` pointers
+----------------------------------------------------------- */
+
+#if MI_PAGE_MAP_FLAT
+
+// flat page-map committed on demand, using one byte per slice (64 KiB).
+// single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses)
+// used by default on <= 40 bit virtual address spaces.
+extern mi_decl_hidden uint8_t* _mi_page_map;
+
+static inline size_t _mi_page_map_index(const void* p) {
+  return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT);
+}
+
+static inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) {
+  const size_t idx = _mi_page_map_index(p);
+  const size_t ofs = _mi_page_map[idx];
+  if (valid != NULL) { *valid = (ofs != 0); }
+  return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT);
+}
+
+static inline mi_page_t* _mi_checked_ptr_page(const void* p) {
+  bool valid;
+  mi_page_t* const page = _mi_ptr_page_ex(p, &valid);
+  return (valid ? page : NULL);
+}
+
+static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {
+  return _mi_ptr_page_ex(p, NULL);
+}
+
+#else
+
+// 2-level page map:
+// double indirection, but low commit and low virtual reserve.
+//
+// the page-map is usually 4 MiB (for 48 bit virtual addresses) and points to sub maps of 64 KiB.
+// the page-map is committed on-demand (in 64 KiB parts) (and sub-maps are committed on-demand as well)
+// one sub page-map = 64 KiB => covers 2^(16-3) * 2^16 = 2^29 = 512 MiB address space
+// the page-map needs 48-(16+13) = 19 bits => 2^19 sub map pointers = 2^22 bytes = 4 MiB reserved size.
+#define MI_PAGE_MAP_SUB_SHIFT     (13)
+#define MI_PAGE_MAP_SUB_COUNT     (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT)
+#define MI_PAGE_MAP_SHIFT         (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)
+#define MI_PAGE_MAP_COUNT         (MI_ZU(1) << MI_PAGE_MAP_SHIFT)
+
+typedef mi_page_t**   mi_submap_t;
+extern mi_decl_hidden _Atomic(mi_submap_t)* _mi_page_map;
+
+static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) {
+  const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE);
+  if (sub_idx != NULL) { *sub_idx = u % MI_PAGE_MAP_SUB_COUNT; }
+  return (u / MI_PAGE_MAP_SUB_COUNT);
+}
+
+static inline mi_submap_t _mi_page_map_at(size_t idx) {
+  return mi_atomic_load_ptr_relaxed(mi_page_t*, &_mi_page_map[idx]);
+}
+
+static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {
+  size_t sub_idx;
+  const size_t idx = _mi_page_map_index(p, &sub_idx);
+  return (_mi_page_map_at(idx))[sub_idx];  // NULL if p==NULL
+}
+
+static inline mi_page_t* _mi_checked_ptr_page(const void* p) {
+  size_t sub_idx;
+  const size_t idx = _mi_page_map_index(p, &sub_idx);
+  mi_submap_t const sub = _mi_page_map_at(idx);
+  if mi_unlikely(sub == NULL) return NULL;
+  return sub[sub_idx];
+}
+
+#endif
+
+
+static inline mi_page_t* _mi_ptr_page(const void* p) {
+  mi_assert_internal(p==NULL || mi_is_in_heap_region(p));
+  #if MI_DEBUG || MI_SECURE || defined(__APPLE__)
+  return _mi_checked_ptr_page(p);
+  #else
+  return _mi_unchecked_ptr_page(p);
+  #endif
+}
+
+
+// Get the block size of a page
+static inline size_t mi_page_block_size(const mi_page_t* page) {
+  mi_assert_internal(page->block_size > 0);
+  return page->block_size;
+}
+
+// Page start
+static inline uint8_t* mi_page_start(const mi_page_t* page) {
+  return page->page_start;
+}
+
+static inline size_t mi_page_size(const mi_page_t* page) {
+  return mi_page_block_size(page) * page->reserved;
+}
+
+static inline uint8_t* mi_page_area(const mi_page_t* page, size_t* size) {
+  if (size) { *size = mi_page_size(page); }
+  return mi_page_start(page);
+}
+
+static inline size_t mi_page_info_size(void) {
+  return _mi_align_up(sizeof(mi_page_t), MI_MAX_ALIGN_SIZE);
+}
+
+static inline bool mi_page_contains_address(const mi_page_t* page, const void* p) {
+  size_t psize;
+  uint8_t* start = mi_page_area(page, &psize);
+  return (start <= (uint8_t*)p && (uint8_t*)p < start + psize);
+}
+
+static inline bool mi_page_is_in_arena(const mi_page_t* page) {
+  return (page->memid.memkind == MI_MEM_ARENA);
+}
+
+static inline bool mi_page_is_singleton(const mi_page_t* page) {
+  return (page->reserved == 1);
+}
+
+// Get the usable block size of a page without fixed padding.
+// This may still include internal padding due to alignment and rounding up size classes.
+static inline size_t mi_page_usable_block_size(const mi_page_t* page) {
+  return mi_page_block_size(page) - MI_PADDING_SIZE;
+}
+
+static inline bool mi_page_meta_is_separated(const mi_page_t* page) {
+  #if MI_PAGE_META_IS_SEPARATED
+  // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL
+  return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN));
+  #else
+  MI_UNUSED(page);
+  return false;  
+  #endif
+}
+
+static inline uint8_t* mi_page_slice_start(const mi_page_t* page) {
+  if (mi_page_meta_is_separated(page)) {  
+    // page meta info is at a separate location (at `arena->pages`)
+    return (uint8_t*)_mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN);
+  }
+  else {
+    // page meta info is at the start of the page slices
+    return (uint8_t*)page;
+  }
+}
+
+// This gives the offset relative to the start slice of a page. 
+static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) {
+  return (page->page_start - mi_page_slice_start(page)) + offset_relative_to_page_start;
+}
+
+// Currently committed part of a page
+static inline size_t mi_page_committed(const mi_page_t* page) {
+  return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - mi_page_slice_offset_of(page,0));
+}
+
+// are all blocks in a page freed?
+// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`.
+static inline bool mi_page_all_free(const mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  return (page->used == 0);
+}
+
+// are there immediately available blocks, i.e. blocks available on the free list.
+static inline bool mi_page_immediate_available(const mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  return (page->free != NULL);
+}
+
+
+// is the page not yet used up to its reserved space?
+static inline bool mi_page_is_expandable(const mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  mi_assert_internal(page->capacity <= page->reserved);
+  return (page->capacity < page->reserved);
+}
+
+
+static inline bool mi_page_is_full(const mi_page_t* page) {
+  const bool full = (page->reserved == page->used);
+  mi_assert_internal(!full || page->free == NULL);
+  return full;
+}
+
+// is more than 7/8th of a page in use?
+static inline bool mi_page_is_mostly_used(const mi_page_t* page) {
+  if (page==NULL) return true;
+  uint16_t frac = page->reserved / 8U;
+  return (page->reserved - page->used <= frac);
+}
+
+// is more than (n-1)/n'th of a page in use?
+static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) {
+  if (page==NULL) return true;
+  uint16_t frac = page->reserved / n;
+  return (page->reserved - page->used <= frac);
+}
+
+
+static inline bool mi_page_is_huge(const mi_page_t* page) {
+  return (mi_page_is_singleton(page) &&
+          (page->block_size > MI_LARGE_MAX_OBJ_SIZE ||
+           (mi_memkind_is_os(page->memid.memkind) && page->memid.mem.os.base < (void*)page)));
+}
+
+static inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t size) {
+  mi_page_queue_t* const pq = &((mi_theap_t*)theap)->pages[_mi_bin(size)];
+  if (size <= MI_LARGE_MAX_OBJ_SIZE) { mi_assert_internal(pq->block_size <= MI_LARGE_MAX_OBJ_SIZE); }
+  return pq;
+}
+
+
+//-----------------------------------------------------------
+// Page thread id and flags
+//-----------------------------------------------------------
+
+// Thread id of thread that owns this page (with flags in the bottom 2 bits)
+static inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) {
+  return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id);
+}
+
+// Plain thread id of the thread that owns this page
+static inline mi_threadid_t mi_page_thread_id(const mi_page_t* page) {
+  return (mi_page_xthread_id(page) & ~MI_PAGE_FLAG_MASK);
+}
+
+static inline mi_page_flags_t mi_page_flags(const mi_page_t* page) {
+  return (mi_page_xthread_id(page) & MI_PAGE_FLAG_MASK);
+}
+
+static inline bool mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) {
+  mi_page_flags_t old;
+  if (set) { old = mi_atomic_or_relaxed(&page->xthread_id, newflag); }
+      else { old = mi_atomic_and_relaxed(&page->xthread_id, ~newflag); }
+  return ((old & newflag) == newflag);
+}
+
+static inline bool mi_page_is_in_full(const mi_page_t* page) {
+  return ((mi_page_flags(page) & MI_PAGE_IN_FULL_QUEUE) != 0);
+}
+
+static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {
+  const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE);
+  if (was_in_full != in_full) {
+    // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220)
+    mi_theap_t* const theap = page->theap;
+    mi_assert_internal(theap!=NULL);
+    if (theap != NULL) {
+      const size_t size = page->capacity * mi_page_block_size(page);
+      if (in_full) { theap->pages_full_size += size; }
+              else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; }
+    }
+  }
+}
+
+static inline bool mi_page_has_interior_pointers(const mi_page_t* page) {
+  return ((mi_page_flags(page) & MI_PAGE_HAS_INTERIOR_POINTERS) != 0);
+}
+
+static inline void mi_page_set_has_interior_pointers(mi_page_t* page, bool has_aligned) {
+  mi_page_flags_set(page, has_aligned, MI_PAGE_HAS_INTERIOR_POINTERS);
+}
+
+static inline void mi_page_set_theap(mi_page_t* page, mi_theap_t* theap) {
+  // mi_assert_internal(!mi_page_is_in_full(page));  // can happen when destroying pages on theap_destroy
+  page->theap = theap;
+  const mi_threadid_t tid = (theap == NULL ? MI_THREADID_ABANDONED : theap->tld->thread_id);
+  mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0);
+
+  // we need to use an atomic cas since a concurrent thread may still set the MI_PAGE_HAS_INTERIOR_POINTERS flag (see `alloc_aligned.c`).
+  mi_threadid_t xtid_old = mi_page_xthread_id(page);
+  mi_threadid_t xtid;
+  do {
+    xtid = tid | (xtid_old & MI_PAGE_FLAG_MASK);
+  } while (!mi_atomic_cas_weak_release(&page->xthread_id, &xtid_old, xtid));
+}
+
+static inline bool mi_page_is_abandoned(const mi_page_t* page) {
+  // note: the xtheap field of an abandoned theap is set to the subproc (for fast reclaim-on-free)
+  return (mi_page_thread_id(page) <= MI_THREADID_ABANDONED_MAPPED);
+}
+
+static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) {
+  return (mi_page_thread_id(page) == MI_THREADID_ABANDONED_MAPPED);
+}
+
+static inline void mi_page_set_abandoned_mapped(mi_page_t* page) {
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_atomic_or_relaxed(&page->xthread_id, (mi_threadid_t)MI_THREADID_ABANDONED_MAPPED);
+}
+
+static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) {
+  mi_assert_internal(mi_page_is_abandoned_mapped(page));
+  mi_atomic_and_relaxed(&page->xthread_id, (mi_threadid_t)MI_PAGE_FLAG_MASK);
+}
+
+
+static inline mi_theap_t* mi_page_theap(const mi_page_t* page) {
+  mi_assert_internal(!mi_page_is_abandoned(page));
+  mi_assert_internal(page->theap != NULL);
+  return page->theap;
+}
+
+static inline mi_tld_t* mi_page_tld(const mi_page_t* page) {
+  mi_assert_internal(!mi_page_is_abandoned(page));
+  mi_assert_internal(page->theap != NULL);
+  return page->theap->tld;
+}
+
+
+static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {
+  mi_heap_t* heap = page->heap;
+  // we use NULL for the main heap to make `_mi_page_get_associated_theap` fast in `free.c:mi_abandoned_page_try_reclaim`.
+  if mi_likely(heap==NULL) heap = mi_heap_main();
+  mi_assert_internal(heap != NULL);
+  return heap;
+}
+
+//-----------------------------------------------------------
+// Thread free list and ownership
+//-----------------------------------------------------------
+
+// Thread free flag helpers
+static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) {
+  return (mi_block_t*)(tf & ~1);
+}
+static inline bool mi_tf_is_owned(mi_thread_free_t tf) {
+  return ((tf & 1) == 1);
+}
+static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) {
+  return (mi_thread_free_t)((uintptr_t)block | (owned ? 1 : 0));
+}
+
+// Thread free access
+static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {
+  return mi_tf_block(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free));
+}
+
+// are there any available blocks?
+static inline bool mi_page_has_any_available(const mi_page_t* page) {
+  mi_assert_internal(page != NULL && page->reserved > 0);
+  return (page->used < page->reserved || (mi_page_thread_free(page) != NULL));
+}
+
+// Owned?
+static inline bool mi_page_is_owned(const mi_page_t* page) {
+  return mi_tf_is_owned(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free));
+}
+
+// get ownership; returns true if the page was not owned before.
+static inline bool mi_page_claim_ownership(mi_page_t* page) {
+  const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, (uintptr_t)1);
+  return ((old&1)==0);
+}
+
+
+/* -------------------------------------------------------------------
+  Guarded objects
+------------------------------------------------------------------- */
+#if MI_GUARDED
+// we always align guarded pointers in a block at an offset
+// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones
+#define MI_BLOCK_TAG_ALIGNED   ((mi_encoded_t)(0))
+#define MI_BLOCK_TAG_GUARDED   (~MI_BLOCK_TAG_ALIGNED)
+#endif
+
+static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) {
+#if MI_GUARDED
+  const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block;
+  return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED);
+#else
+  MI_UNUSED(block); MI_UNUSED(p);
+  return false;
+#endif  
+}
+
+#if MI_GUARDED
+static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) {
+  // this code is written to result in fast assembly as it is on the hot path for allocation
+  const size_t count = theap->guarded_sample_count - 1;  // if the rate was 0, this will underflow and count for a long time..
+  if mi_likely(count != 0) {
+    // no sample
+    theap->guarded_sample_count = count;
+    return false;
+  }
+  else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) {
+    // use guarded allocation
+    theap->guarded_sample_count = theap->guarded_sample_rate;  // reset
+    return (theap->guarded_sample_rate != 0);
+  }
+  else {
+    // failed size criteria, rewind count (but don't write to an empty theap)
+    if (theap->guarded_sample_rate != 0) { theap->guarded_sample_count = 1; }
+    return false;
+  }
+}
+
+mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;
+
+#endif
+
+
+/* -------------------------------------------------------------------
+Encoding/Decoding the free list next pointers
+
+This is to protect against buffer overflow exploits where the
+free list is mutated. Many hardened allocators xor the next pointer `p`
+with a secret key `k1`, as `p^k1`. This prevents overwriting with known
+values but might be still too weak: if the attacker can guess
+the pointer `p` this  can reveal `k1` (since `p^k1^p == k1`).
+Moreover, if multiple blocks can be read as well, the attacker can
+xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot
+about the pointers (and subsequently `k1`).
+
+Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<<k1)+k1`.
+Since these operations are not associative, the above approaches do not
+work so well any more even if the `p` can be guesstimated. For example,
+for the read case we can subtract two entries to discard the `+k1` term,
+but that leads to `((p1^k2)<<<k1) - ((p2^k2)<<<k1)` at best.
+We include the left-rotation since xor and addition are otherwise linear
+in the lowest bit. Finally, both keys are unique per page which reduces
+the re-use of keys by a large factor.
+
+We also pass a separate `null` value to be used as `NULL` or otherwise
+`(k2<<<k1)+k1` would appear (too) often as a sentinel value.
+------------------------------------------------------------------- */
+
+static inline bool mi_is_in_same_page(const void* p, const void* q) {
+  mi_page_t* page = _mi_ptr_page(p);
+  return mi_page_contains_address(page,q);
+  // return (_mi_ptr_page(p) == _mi_ptr_page(q));
+}
+
+static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) {
+  void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]);
+  return (p==null ? NULL : p);
+}
+
+static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) {
+  uintptr_t x = (uintptr_t)(p==NULL ? null : p);
+  return mi_rotl(x ^ keys[1], keys[0]) + keys[0];
+}
+
+static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) {
+  const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys));
+  // make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951)
+  #if MI_BIG_ENDIAN
+  return (x & 0x00FFFFFF);
+  #else
+  return (x & 0xFFFFFF00);
+  #endif
+}
+
+static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) {
+  mi_track_mem_defined(block,sizeof(mi_block_t));
+  mi_block_t* next;
+  #if MI_ENCODE_FREELIST
+  next = (mi_block_t*)mi_ptr_decode(null, block->next, keys);
+  #else
+  MI_UNUSED(keys); MI_UNUSED(null);
+  next = (mi_block_t*)block->next;
+  #endif
+  mi_track_mem_noaccess(block,sizeof(mi_block_t));
+  return next;
+}
+
+static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) {
+  mi_track_mem_undefined(block,sizeof(mi_block_t));
+  #if MI_ENCODE_FREELIST
+  block->next = mi_ptr_encode(null, next, keys);
+  #else
+  MI_UNUSED(keys); MI_UNUSED(null);
+  block->next = (mi_encoded_t)next;
+  #endif
+  mi_track_mem_noaccess(block,sizeof(mi_block_t));
+}
+
+static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) {
+  #if MI_ENCODE_FREELIST
+  mi_block_t* next = mi_block_nextx(page,block,page->keys);
+  // check for free list corruption: is `next` at least in the same page?
+  // TODO: check if `next` is `page->block_size` aligned?
+  if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) {
+    _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next);
+    next = NULL;
+  }
+  return next;
+  #else
+  MI_UNUSED(page);
+  return mi_block_nextx(page,block,NULL);
+  #endif
+}
+
+static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) {
+  #if MI_ENCODE_FREELIST
+  mi_block_set_nextx(page,block,next, page->keys);
+  #else
+  MI_UNUSED(page);
+  mi_block_set_nextx(page,block,next,NULL);
+  #endif
+}
+
+/* -----------------------------------------------------------
+  arena blocks
+----------------------------------------------------------- */
+
+// Blocks needed for a given byte size
+static inline size_t mi_slice_count_of_size(size_t size) {
+  return _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
+}
+
+// Byte size of a number of blocks
+static inline size_t mi_size_of_slices(size_t bcount) {
+  return (bcount * MI_ARENA_SLICE_SIZE);
+}
+
+
+/* -----------------------------------------------------------
+  memory id's
+----------------------------------------------------------- */
+
+static inline mi_memid_t _mi_memid_create(mi_memkind_t memkind) {
+  mi_memid_t memid;
+  _mi_memzero_var(memid);
+  memid.memkind = memkind;
+  return memid;
+}
+
+static inline mi_memid_t _mi_memid_none(void) {
+  return _mi_memid_create(MI_MEM_NONE);
+}
+
+static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool committed, bool is_zero, bool is_large) {
+  mi_memid_t memid = _mi_memid_create(MI_MEM_OS);
+  memid.mem.os.base = base;
+  memid.mem.os.size = size;
+  memid.initially_committed = committed;
+  memid.initially_zero = is_zero;
+  memid.is_pinned = is_large;
+  return memid;
+}
+
+static inline mi_memid_t _mi_memid_create_meta(void* mpage, size_t block_idx, size_t block_count) {
+  mi_memid_t memid = _mi_memid_create(MI_MEM_META);
+  memid.mem.meta.meta_page = mpage;
+  memid.mem.meta.block_index = (uint32_t)block_idx;
+  memid.mem.meta.block_count = (uint32_t)block_count;
+  memid.initially_committed = true;
+  memid.initially_zero = true;
+  memid.is_pinned = true;
+  return memid;
+}
+
+
+// -------------------------------------------------------------------
+// Fast "random" shuffle
+// -------------------------------------------------------------------
+
+static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
+  if (x==0) { x = 17; }   // ensure we don't get stuck in generating zeros
+#if (MI_INTPTR_SIZE>=8)
+  // by Sebastiano Vigna, see: <http://xoshiro.di.unimi.it/splitmix64.c>
+  x ^= x >> 30;
+  x *= 0xbf58476d1ce4e5b9UL;
+  x ^= x >> 27;
+  x *= 0x94d049bb133111ebUL;
+  x ^= x >> 31;
+#elif (MI_INTPTR_SIZE==4)
+  // by Chris Wellons, see: <https://nullprogram.com/blog/2018/07/31/>
+  x ^= x >> 16;
+  x *= 0x7feb352dUL;
+  x ^= x >> 15;
+  x *= 0x846ca68bUL;
+  x ^= x >> 16;
+#endif
+  return x;
+}
+
+
+// ---------------------------------------------------------------------------------
+// Provide our own `_mi_memcpy/set` for potential performance optimizations.
+//
+// For now, only on x64/x86 we optimize to `rep movsb/stosb`.
+// Generally, we check for "fast short rep movsb/stosb" (FSRM/FSRS) or "fast enhanced rep movsb" (ERMS) support
+// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253.
+// Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows.
+// ---------------------------------------------------------------------------------
+
+#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__))
+
+extern mi_decl_hidden size_t _mi_cpu_movsb_max;  // in init.c
+extern mi_decl_hidden size_t _mi_cpu_stosb_max;
+
+static inline void mi_rep_movsb(void* dst, const void* src, size_t n) {
+  #if defined(__GNUC__)
+  __asm volatile("rep movsb" : "+D"(dst), "+c"(n), "+S"(src) : : "memory");
+  #else
+  __movsb((unsigned char*)dst, (const unsigned char*)src, n);
+  #endif
+}
+
+static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) {
+  #if defined(__GNUC__)
+  __asm volatile("rep stosb" : "+D"(dst), "+c"(n) : "a"(val) : "memory");
+  #else
+  __stosb((unsigned char*)dst, val, n);
+  #endif
+}
+
+static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
+  if mi_likely(n <= _mi_cpu_movsb_max) {  // has fsrm && n <= 127  (todo: and maybe has erms?)
+    mi_rep_movsb(dst, src, n);
+  }
+  else {
+    memcpy(dst, src, n);
+  }
+}
+
+static inline void _mi_memset(void* dst, int val, size_t n) {
+  if mi_likely(n <= _mi_cpu_stosb_max) {  // has fsrs && n <= 127
+    mi_rep_stosb(dst, (uint8_t)val, n);
+  }
+  else {
+    memset(dst, val, n);
+  }
+}
+
+#else
+
+static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
+  memcpy(dst, src, n);
+}
+
+static inline void _mi_memset(void* dst, int val, size_t n) {
+  memset(dst, val, n);
+}
+
+#endif
+
+// -------------------------------------------------------------------------------
+// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned
+// This is used for example in `mi_realloc`.
+// -------------------------------------------------------------------------------
+
+#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
+
+// On GCC/CLang we provide a hint that the pointers are word aligned.
+static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
+  mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
+  void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
+  const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);
+  _mi_memcpy(adst, asrc, n);
+}
+
+static inline void _mi_memset_aligned(void* dst, int val, size_t n) {
+  mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
+  void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
+  _mi_memset(adst, val, n);
+}
+
+#else
+
+// Default fallback on `_mi_memcpy`
+static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
+  mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
+  _mi_memcpy(dst, src, n);
+}
+
+static inline void _mi_memset_aligned(void* dst, int val, size_t n) {
+  mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
+  _mi_memset(dst, val, n);
+}
+
+#endif
+
+static inline void _mi_memzero(void* dst, size_t n) {
+  _mi_memset(dst, 0, n);
+}
+
+static inline void _mi_memzero_aligned(void* dst, size_t n) {
+  _mi_memset_aligned(dst, 0, n);
+}
+
+
+
+#endif  // MI_INTERNAL_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/prim.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/prim.h
new file mode 100644
index 00000000..aecfc807
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/prim.h
@@ -0,0 +1,542 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MIMALLOC_PRIM_H
+#define MIMALLOC_PRIM_H
+#include "internal.h"             // mi_decl_hidden
+
+// --------------------------------------------------------------------------
+// This file specifies the primitive portability API.
+// Each OS/host needs to implement these primitives, see `src/prim`
+// for implementations on Window, macOS, WASI, and Linux/Unix.
+//
+// note: on all primitive functions, we always have result parameters != NULL, and:
+//  addr != NULL and page aligned
+//  size > 0     and page aligned
+//  the return value is an error code as an `int` where 0 is success
+// --------------------------------------------------------------------------
+
+// OS memory configuration
+typedef struct mi_os_mem_config_s {
+  size_t  page_size;              // default to 4KiB
+  size_t  large_page_size;        // 0 if not supported, usually 2MiB (4MiB on Windows)
+  size_t  alloc_granularity;      // smallest allocation size (usually 4KiB, on Windows 64KiB)
+  size_t  physical_memory_in_kib; // physical memory size in KiB
+  size_t  virtual_address_bits;   // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization)
+  bool    has_overcommit;         // can we reserve more memory than can be actually committed?
+  bool    has_partial_free;       // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc)
+  bool    has_virtual_reserve;    // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)
+  bool    has_transparent_huge_pages;  // true if transparent huge pages are enabled (on Linux)
+} mi_os_mem_config_t;
+
+// Initialize
+void _mi_prim_mem_init( mi_os_mem_config_t* config );
+
+// Free OS memory
+int _mi_prim_free(void* addr, size_t size );
+
+// Allocate OS memory. Return NULL on error.
+// The `try_alignment` is just a hint and the returned pointer does not have to be aligned.
+// If `commit` is false, the virtual memory range only needs to be reserved (with no access)
+// which will later be committed explicitly using `_mi_prim_commit`.
+// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
+// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored.
+// pre: !commit => !allow_large
+//      try_alignment >= _mi_os_page_size() and a power of 2
+int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr);
+
+// Commit memory. Returns error code or 0 on success.
+// For example, on Linux this would make the memory PROT_READ|PROT_WRITE.
+// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows)
+int _mi_prim_commit(void* addr, size_t size, bool* is_zero);
+
+// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true
+// if the memory would need to be re-committed. For example, on Windows this is always true,
+// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit.
+// pre: needs_recommit != NULL
+int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit);
+
+// Reset memory. The range keeps being accessible but the content might be reset to zero at any moment.
+// Returns error code or 0 on success.
+int _mi_prim_reset(void* addr, size_t size);
+
+// Reuse memory. This is called for memory that is already committed but
+// may have been reset (`_mi_prim_reset`) or decommitted (`_mi_prim_decommit`) where `needs_recommit` was false.
+// Returns error code or 0 on success. On most platforms this is a no-op.
+int _mi_prim_reuse(void* addr, size_t size);
+
+// Protect memory. Returns error code or 0 on success.
+int _mi_prim_protect(void* addr, size_t size, bool protect);
+
+// Allocate huge (1GiB) pages possibly associated with a NUMA node.
+// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
+// pre: size > 0  and a multiple of 1GiB.
+//      numa_node is either negative (don't care), or a numa node number.
+int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);
+
+// Return the current NUMA node
+size_t _mi_prim_numa_node(void);
+
+// Return the number of logical NUMA nodes
+size_t _mi_prim_numa_node_count(void);
+
+// Clock ticks
+mi_msecs_t _mi_prim_clock_now(void);
+
+// Return process information (only for statistics)
+typedef struct mi_process_info_s {
+  mi_msecs_t  elapsed;
+  mi_msecs_t  utime;
+  mi_msecs_t  stime;
+  size_t      current_rss;
+  size_t      peak_rss;
+  size_t      current_commit;
+  size_t      peak_commit;
+  size_t      page_faults;
+} mi_process_info_t;
+
+void _mi_prim_process_info(mi_process_info_t* pinfo);
+
+// Default stderr output. (only for warnings etc. with verbose enabled)
+// msg != NULL && _mi_strlen(msg) > 0
+void _mi_prim_out_stderr( const char* msg );
+
+// Get an environment variable. (only for options)
+// name != NULL, result != NULL, result_size >= 64
+bool _mi_prim_getenv(const char* name, char* result, size_t result_size);
+
+
+// Fill a buffer with strong randomness; return `false` on error or if
+// there is no strong randomization available.
+bool _mi_prim_random_buf(void* buf, size_t buf_len);
+
+// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination.
+void _mi_prim_thread_init_auto_done(void);
+
+// Called on process exit and may take action to clean up resources associated with the thread auto done.
+void _mi_prim_thread_done_auto_done(void);
+
+// Called when the default theap for a thread changes
+void _mi_prim_thread_associate_default_theap(mi_theap_t* theap);
+
+// Is this thread part of a thread pool?
+bool _mi_prim_thread_is_in_threadpool(void);
+
+// Yield to other threads. Should be similar to `sleep(0)`. 
+// Is called only in rare situations and does not have to be lightning fast.
+void _mi_prim_thread_yield(void);
+
+//-------------------------------------------------------------------
+// Access to TLS (thread local storage) slots.
+// We need fast access to both a unique thread id (in `free.c:mi_free`) and
+// to a thread-local theap pointer (in `alloc.c:mi_malloc`).
+// To achieve this we use specialized code for various platforms.
+//-------------------------------------------------------------------
+
+// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.
+// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform.
+// If you test on another platform and it works please send a PR :-)
+// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.
+//
+// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly,
+// but unfortunately we can not detect support reliably (see issue #883)
+// We also use it on Apple OS as we use a TLS slot for the default theap there.
+#if (defined(_WIN32)) || \
+    (defined(__GNUC__) && ( \
+           (defined(__GLIBC__)   && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
+        || (defined(__APPLE__)   && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \
+        || (defined(__BIONIC__)  && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
+        || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
+        || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
+      ))
+
+static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {
+  void* res;
+  const size_t ofs = (slot*sizeof(void*));
+  #if defined(_WIN32)
+    #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)
+      res = (void*)__readgsqword((unsigned long)ofs);   // direct load at offset from gs
+    #elif _M_IX86 && !defined(_M_ARM64EC)
+      res = (void*)__readfsdword((unsigned long)ofs);   // direct load at offset from fs
+    #else
+      res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs);
+    #endif
+  #elif defined(__i386__)
+    __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : );  // x86 32-bit always uses GS
+  #elif defined(__APPLE__) && defined(__x86_64__)
+    __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : );  // x86_64 macOSX uses GS
+  #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
+    __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : );  // x32 ABI
+  #elif defined(__x86_64__)
+    __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : );  // x86_64 Linux, BSD uses FS
+  #elif defined(__arm__)
+    void** tcb; MI_UNUSED(ofs);
+    __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
+    res = tcb[slot];
+  #elif defined(__aarch64__)
+    void** tcb; MI_UNUSED(ofs);
+    #if defined(__APPLE__) // M1, issue #343
+    __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
+    #else
+    __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
+    #endif
+    res = tcb[slot];
+  #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
+    MI_UNUSED(ofs);
+    res = pthread_getspecific(slot);
+  #else
+    #define MI_HAS_TLS_SLOT 0
+    MI_UNUSED(ofs);
+    res = NULL;
+  #endif
+  return res;
+}
+
+#ifndef MI_HAS_TLS_SLOT
+#define MI_HAS_TLS_SLOT 1
+#endif
+
+// setting a tls slot is only used on macOS for now
+static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
+  const size_t ofs = (slot*sizeof(void*));
+  #if defined(_WIN32)
+    ((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs);
+  #elif defined(__i386__)
+    __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : );  // 32-bit always uses GS
+  #elif defined(__APPLE__) && defined(__x86_64__)
+    __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : );  // x86_64 macOS uses GS
+  #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
+    __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : );  // x32 ABI
+  #elif defined(__x86_64__)
+    __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : );  // x86_64 Linux, BSD uses FS
+  #elif defined(__arm__)
+    void** tcb; MI_UNUSED(ofs);
+    __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
+    tcb[slot] = value;
+  #elif defined(__aarch64__)
+    void** tcb; MI_UNUSED(ofs);
+    #if defined(__APPLE__) // M1, issue #343
+    __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
+    #else
+    __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
+    #endif
+    tcb[slot] = value;
+  #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
+    MI_UNUSED(ofs);
+    pthread_setspecific(slot, value);
+  #else
+    MI_UNUSED(ofs); MI_UNUSED(value);
+  #endif
+}
+
+#endif
+
+
+// defined in `init.c`; do not use these directly
+extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main;     // theap belonging to the main heap
+extern mi_decl_hidden bool _mi_process_is_initialized;                // has mi_process_init been called?
+
+
+//-------------------------------------------------------------------
+// Get a fast unique thread id.
+//
+// Getting the thread id should be performant as it is called in the
+// fast path of `_mi_free` and we specialize for various platforms as
+// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.
+// We only require _mi_prim_thread_id() to return a unique id
+// for each thread (unequal to zero).
+//-------------------------------------------------------------------
+
+
+// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id
+// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
+// Nevertheless, it seems needed on older graviton platforms (see issue #851).
+// For now, we only enable this for specific platforms.
+#if !defined(MI_USE_BUILTIN_THREAD_POINTER)   /* allow user override */
+  #if !defined(__APPLE__)  /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
+      && !defined(__CYGWIN__) \
+      && !defined(MI_LIBC_MUSL) \
+      && (!defined(__clang_major__) || __clang_major__ >= 14)  /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
+    #if    (defined(__GNUC__) && (__GNUC__ >= 7)  && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
+        || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
+        || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
+      #define MI_USE_BUILTIN_THREAD_POINTER  1
+    #endif
+  #endif
+#endif
+
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept;
+
+static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
+  const mi_threadid_t tid = __mi_prim_thread_id();
+  mi_assert_internal(tid > 1);
+  mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0);  // bottom 2 bits are clear?
+  return tid;
+}
+
+// Get a unique id for the current thread.
+#if defined(MI_PRIM_THREAD_ID)
+
+static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
+  const mi_threadid_t tid = MI_PRIM_THREAD_ID();  // used for example by CPython for a free threaded build (see python/cpython#115488)
+  mi_assert_internal( (tid & 0x03) == 0 );        // mimalloc reserves the bottom 2 bits
+  return tid;
+}
+
+#elif defined(_WIN32)
+
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
+  // Windows: works on Intel and ARM in both 32- and 64-bit
+  return (uintptr_t)NtCurrentTeb();
+}
+
+#elif MI_USE_BUILTIN_THREAD_POINTER
+
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
+  // Works on most Unix based platforms with recent compilers
+  return (uintptr_t)__builtin_thread_pointer();
+}
+
+#elif MI_HAS_TLS_SLOT
+
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
+  #if defined(__BIONIC__)
+    // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
+    // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
+    return (uintptr_t)mi_prim_tls_slot(1);
+  #else
+    // in all our other targets, slot 0 is the thread id
+    // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h
+    // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36
+    return (uintptr_t)mi_prim_tls_slot(0);
+  #endif
+}
+
+#else
+
+// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
+  return (uintptr_t)&__mi_theap_main;
+}
+
+#endif
+
+
+
+/* ----------------------------------------------------------------------------------------
+Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`).
+
+This is inlined here as it is on the fast path for allocation functions.
+We have 4 models:
+
+- MI_TLS_MODEL_THREAD_LOCAL: use regular thread local (default on Linux, FreeBSD, etc)
+    On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a
+    thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures
+    that the storage will always be available and properly initialized (with an empty theap).
+
+    On some platforms the underlying TLS implementation (or the loader) will call itself `malloc`
+    on a first access to a thread local and recurse in the MI_TLS_MODEL_THREAD_LOCAL.
+    A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process
+    is initialized before accessing the thread-local. This is a check in the fast path though
+    so this should be avoided.
+
+- MI_TLS_MODEL_FIXED_SLOT: use a fixed slot in the TLS block (default on macOS)
+    This reserves an unused and fixed TLS slot. This is fast and avoids the problem
+    where the underlying TLS implementation (or the loader) will call itself `malloc`
+    on a first access to a thread local (and recurse in the MI_TLS_MODEL_THREAD_LOCAL).
+    This goes wrong though if the OS or a library uses the same fixed slot.
+
+- MI_TLS_MODEL_DYNAMIC_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows)
+    Windows has somewhat slow thread locals so by default we use TlsAlloc'd slots which
+    can be more efficient. First tries to use one of the "direct" first 64 slots which 
+    are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots).
+    (If the allocated slot happens to always be under 64 for a particular program,
+    one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.)
+
+- MI_TLS_MODEL_DYNAMIC_PTHREADS: use `pthread_getspecific`. (default on OpenBSD, maybe good for Android as well?)
+    Use pthread local storage. Somewhat slow but can work well depending on the platform.
+
+Each model should define `MI_THEAP_INITASNULL` to signify that the initial value
+returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap).
+This incurs an extra check in the fast path (but can often be combined in an existing check).
+------------------------------------------------------------------------------------------- */
+
+static inline mi_theap_t* _mi_theap_default(void);
+static inline mi_theap_t* _mi_theap_cached(void);
+
+#if defined(_WIN32)
+  #define MI_TLS_MODEL_DYNAMIC_WIN32        1    
+#elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__)  // macOS on arm64 or x64
+  // #define MI_TLS_MODEL_DYNAMIC_PTHREADS  1    // also works but a bit slower
+  #define MI_TLS_MODEL_FIXED_SLOT           1
+  #define MI_TLS_MODEL_FIXED_SLOT_DEFAULT   108  // seems unused. @apple: it would be great to get 2 official slots for custom allocators :-)
+  #define MI_TLS_MODEL_FIXED_SLOT_CACHED    109
+  // we used before __PTK_FRAMEWORK_OLDGC_KEY9 (89) but that seems used now.
+  // see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
+#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__)
+  #define MI_TLS_MODEL_DYNAMIC_PTHREADS     1
+  // #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL  1
+#else
+  #define MI_TLS_MODEL_THREAD_LOCAL         1
+#endif
+
+// Declared this way to optimize register spills and branches
+mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void);
+
+static inline mi_theap_t* __mi_theap_empty(void) {
+  #if __GNUC__
+  __asm("");  // prevent conditional load
+  return (mi_theap_t*)&_mi_theap_empty;
+  #else
+  return _mi_theap_empty_get();
+  #endif
+}
+
+#if MI_TLS_MODEL_THREAD_LOCAL
+// Thread local with an initial value (default on Linux). Very efficient.
+
+extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default;  // default theap to allocate from
+extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached;   // theap from the last used heap
+
+static inline mi_theap_t* _mi_theap_default(void) {
+  #if defined(MI_TLS_RECURSE_GUARD)
+  if (mi_unlikely(!_mi_process_is_initialized)) return _mi_theap_empty_get();
+  #endif
+  return __mi_theap_default;
+}
+
+static inline mi_theap_t* _mi_theap_cached(void) {
+  return __mi_theap_cached;
+}
+
+#elif MI_TLS_MODEL_FIXED_SLOT
+// Fixed TLS slot (default on macOS).
+#define MI_THEAP_INITASNULL  1
+
+static inline mi_theap_t* _mi_theap_default(void) {
+  return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_DEFAULT);
+}
+
+static inline mi_theap_t* _mi_theap_cached(void) {
+  return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_CACHED);
+}
+
+#elif MI_TLS_MODEL_DYNAMIC_WIN32
+// Dynamic TLS slot (default on Windows)
+#define MI_THEAP_INITASNULL  1
+
+// We try to use direct slots (64), but can also use the expansion slots (upto 1024 extra available)
+// See <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm> for the offsets.
+#if MI_SIZE_SIZE==4
+#define MI_TLS_EXPANSION_SLOT    (0x0F94 / MI_SIZE_SIZE)
+#else
+#define MI_TLS_EXPANSION_SLOT    (0x1780 / MI_SIZE_SIZE)
+#endif
+
+extern mi_decl_hidden size_t _mi_theap_default_slot;
+extern mi_decl_hidden size_t _mi_theap_cached_slot;
+extern mi_decl_hidden size_t _mi_theap_default_expansion_slot;
+extern mi_decl_hidden size_t _mi_theap_cached_expansion_slot;
+
+static inline mi_theap_t* _mi_theap_default(void) {
+  const size_t slot = _mi_theap_default_slot;
+  mi_theap_t* theap  = (mi_theap_t*)mi_prim_tls_slot(slot);
+  #if !MI_WIN_DIRECT_TLS
+  if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
+    if mi_likely(theap!=NULL) {                 // initialized (on this thread)?
+      theap = ((mi_theap_t**)theap)[_mi_theap_default_expansion_slot];
+    }
+  }
+  #endif
+  return theap;
+}
+
+static inline mi_theap_t* _mi_theap_cached(void) {
+  const size_t slot = _mi_theap_cached_slot;
+  mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
+  #if !MI_WIN_DIRECT_TLS
+  if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
+    if mi_likely(theap!=NULL) {                 // initialized (on this thread)?
+      theap = ((mi_theap_t**)theap)[_mi_theap_cached_expansion_slot];
+    }
+  }
+  #endif
+  return theap;
+}
+
+#elif MI_TLS_MODEL_DYNAMIC_PTHREADS
+// Dynamic pthread slot on less common platforms. This is not too bad. (default on OpenBSD)
+#define MI_THEAP_INITASNULL  1
+
+extern mi_decl_hidden pthread_key_t _mi_theap_default_key;
+extern mi_decl_hidden pthread_key_t _mi_theap_cached_key;
+
+static inline mi_theap_t* _mi_theap_default(void) {
+  #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
+  // we can skip this check if using the initial key will return NULL from pthread_getspecific
+  if mi_unlikely(_mi_theap_default_key==0) { return NULL; }
+  #endif
+  return (mi_theap_t*)pthread_getspecific(_mi_theap_default_key);
+}
+
+static inline mi_theap_t* _mi_theap_cached(void) {
+  #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
+  // we can skip this check if using the initial key will return NULL from pthread_getspecific
+  if mi_unlikely(_mi_theap_cached_key==0) { return NULL; }
+  #endif
+  return (mi_theap_t*)pthread_getspecific(_mi_theap_cached_key);
+}
+
+#else
+#error "no TLS model is defined for this platform?"
+#endif
+
+
+// Check if a thread is initialized (without using a thread-local if using fixed slots)
+static inline bool _mi_thread_is_initialized(void) {
+  return (mi_theap_is_initialized(_mi_theap_default()));
+}
+
+// Get (and possible create) the theap belonging to a heap
+// We cache the last accessed theap in `_mi_theap_cached` for better performance.
+static inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) {
+  mi_theap_t* theap = _mi_theap_cached();  
+  #if MI_THEAP_INITASNULL
+  if mi_likely(theap!=NULL && _mi_theap_heap(theap)==heap) return theap;
+  #else
+  if mi_likely(_mi_theap_heap(theap)==heap) return theap;
+  #endif
+  return _mi_heap_theap_get_or_init(heap);
+}
+
+// Get the theap belonging to a heap without creating in if it is not yet initialized.
+static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {
+  mi_theap_t* theap = _mi_theap_cached();
+  #if MI_THEAP_INITASNULL
+  if mi_unlikely(theap==NULL || _mi_theap_heap(theap)!=heap)
+  #else
+  if mi_unlikely(_mi_theap_heap(theap)!=heap)
+  #endif
+  {
+    theap = _mi_heap_theap_get_peek(heap);  // don't update the cache on a query (?)
+  }
+  mi_assert(theap==NULL || _mi_theap_heap(theap)==heap);
+  return theap;
+}
+
+// Find the associated theap or NULL if it does not exist (during shutdown)
+// Should be fast as it is called in `free.c:mi_free_try_collect`.
+static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) {
+  mi_heap_t* const heap = page->heap;
+  mi_theap_t* theap;
+  if mi_likely(heap==NULL) { theap = __mi_theap_main; }  // note: on macOS accessing the thread_local can cause allocation during thread shutdown (and reinitialize the thread)!
+                      else { theap = _mi_heap_theap_peek(heap); }
+  mi_assert_internal(theap==NULL || _mi_thread_id()==theap->tld->thread_id);
+  return theap;
+}
+
+#endif  // MI_PRIM_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/track.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/track.h
new file mode 100644
index 00000000..1f402129
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/track.h
@@ -0,0 +1,150 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MI_TRACK_H
+#define MI_TRACK_H
+
+/* ------------------------------------------------------------------------------------------------------
+Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers.
+These can be defined for tracking allocation:
+
+  #define mi_track_malloc_size(p,reqsize,size,zero)
+  #define mi_track_free_size(p,_size)
+
+The macros are set up such that the size passed to `mi_track_free_size`
+always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`).
+The `reqsize` is what the user requested, and `size >= reqsize`.
+The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled,
+or otherwise it is the usable block size which may be larger than the original request.
+Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc).
+The `zero` parameter is `true` if the allocated block is zero initialized.
+
+Optional:
+
+  #define mi_track_align(p,alignedp,offset,size)
+  #define mi_track_resize(p,oldsize,newsize)
+  #define mi_track_init()
+
+The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block.
+The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`).
+The `mi_track_resize` is currently unused but could be called on reallocations within a block.
+`mi_track_init` is called at program start.
+
+The following macros are for tools like asan and valgrind to track whether memory is
+defined, undefined, or not accessible at all:
+
+  #define mi_track_mem_defined(p,size)
+  #define mi_track_mem_undefined(p,size)
+  #define mi_track_mem_noaccess(p,size)
+
+-------------------------------------------------------------------------------------------------------*/
+
+#if MI_TRACK_VALGRIND
+// valgrind tool
+
+#define MI_TRACK_ENABLED      1
+#define MI_TRACK_HEAP_DESTROY 1           // track free of individual blocks on theap_destroy
+#define MI_TRACK_TOOL         "valgrind"
+
+#include <valgrind/valgrind.h>
+#include <valgrind/memcheck.h>
+
+#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero)
+#define mi_track_free_size(p,_size)               VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/)
+#define mi_track_resize(p,oldsize,newsize)        VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/)
+#define mi_track_mem_defined(p,size)              VALGRIND_MAKE_MEM_DEFINED(p,size)
+#define mi_track_mem_undefined(p,size)            VALGRIND_MAKE_MEM_UNDEFINED(p,size)
+#define mi_track_mem_noaccess(p,size)             VALGRIND_MAKE_MEM_NOACCESS(p,size)
+
+#elif MI_TRACK_ASAN
+// address sanitizer
+
+#define MI_TRACK_ENABLED      1
+#define MI_TRACK_HEAP_DESTROY 0
+#define MI_TRACK_TOOL         "asan"
+
+#include <sanitizer/asan_interface.h>
+
+#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size)
+#define mi_track_free_size(p,size)                ASAN_POISON_MEMORY_REGION(p,size)
+#define mi_track_mem_defined(p,size)              ASAN_UNPOISON_MEMORY_REGION(p,size)
+#define mi_track_mem_undefined(p,size)            ASAN_UNPOISON_MEMORY_REGION(p,size)
+#define mi_track_mem_noaccess(p,size)             ASAN_POISON_MEMORY_REGION(p,size)
+
+#elif MI_TRACK_ETW
+// windows event tracing
+
+#define MI_TRACK_ENABLED      1
+#define MI_TRACK_HEAP_DESTROY 1
+#define MI_TRACK_TOOL         "ETW"
+
+#include "../src/prim/windows/etw.h"
+
+#define mi_track_init()                           EventRegistermicrosoft_windows_mimalloc()
+#define mi_track_done()                           EventUnregistermicrosoft_windows_mimalloc()
+#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size)
+#define mi_track_free_size(p,size)                EventWriteETW_MI_FREE((UINT64)(p), size)
+
+#else
+// no tracking
+
+#define MI_TRACK_ENABLED      0
+#define MI_TRACK_HEAP_DESTROY 0
+#define MI_TRACK_TOOL         "none"
+
+#define mi_track_malloc_size(p,reqsize,size,zero)
+#define mi_track_free_size(p,_size)
+
+#endif
+
+// -------------------
+// Utility definitions
+
+#ifndef mi_track_resize
+#define mi_track_resize(p,oldsize,newsize)      do{ mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false); } while(0)
+#endif
+
+#ifndef mi_track_align
+#define mi_track_align(p,alignedp,offset,size)  mi_track_mem_noaccess(p,offset)
+#endif
+
+#ifndef mi_track_init
+#define mi_track_init()
+#endif
+
+#ifndef mi_track_done
+#define mi_track_done()
+#endif
+
+#ifndef mi_track_mem_defined
+#define mi_track_mem_defined(p,size)
+#endif
+
+#ifndef mi_track_mem_undefined
+#define mi_track_mem_undefined(p,size)
+#endif
+
+#ifndef mi_track_mem_noaccess
+#define mi_track_mem_noaccess(p,size)
+#endif
+
+
+#if MI_PADDING
+#define mi_track_malloc(p,reqsize,zero) \
+  if ((p)!=NULL) { \
+    mi_assert_internal(mi_usable_size(p)==(reqsize)); \
+    mi_track_malloc_size(p,reqsize,reqsize,zero); \
+  }
+#else
+#define mi_track_malloc(p,reqsize,zero) \
+  if ((p)!=NULL) { \
+    mi_assert_internal(mi_usable_size(p)>=(reqsize)); \
+    mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \
+  }
+#endif
+
+#endif // MI_TRACK_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/types.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/types.h
new file mode 100644
index 00000000..31fdace9
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/types.h
@@ -0,0 +1,732 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MI_TYPES_H
+#define MI_TYPES_H
+
+// --------------------------------------------------------------------------
+// This file contains the main type definitions for mimalloc:
+// mi_heap_t      : all data for a heap; usually there is just one main default heap.
+// mi_theap_t     : a thread local heap belonging to a specific heap:
+//                  maintains lists of thread-local heap pages that have free space.
+// mi_page_t      : a mimalloc page (usually 64KiB or 512KiB) from
+//                  where objects of a single size are allocated.
+//                  Note: we write "OS page" for OS memory pages while
+//                  using plain "page" for mimalloc pages (`mi_page_t`).
+// mi_arena_t     : a large memory area where pages are allocated (process shared)
+// mi_tld_t       : thread local data
+// mi_subproc_t   : all heaps belong to a sub-process (usually just the main one)
+// --------------------------------------------------------------------------
+
+
+#include <mimalloc-stats.h>
+#include <stddef.h>   // ptrdiff_t
+#include <stdint.h>   // uintptr_t, uint16_t, etc
+#include <stdbool.h>  // bool
+#include <limits.h>   // SIZE_MAX etc.
+#include <errno.h>    // error codes
+#include "bits.h"     // size defines (MI_INTPTR_SIZE etc), bit operations
+#include "atomic.h"   // _Atomic primitives
+
+// Minimal alignment necessary. On most platforms 16 bytes are needed
+// due to SSE registers for example. This must be at least `sizeof(void*)`
+#ifndef MI_MAX_ALIGN_SIZE
+#define MI_MAX_ALIGN_SIZE  16   // sizeof(max_align_t)
+#endif
+
+
+// ------------------------------------------------------
+// Variants
+// ------------------------------------------------------
+
+// Define NDEBUG in the release version to disable assertions.
+// #define NDEBUG
+
+// Define MI_TRACK_<tool> to enable tracking support
+// #define MI_TRACK_VALGRIND 1
+// #define MI_TRACK_ASAN     1
+// #define MI_TRACK_ETW      1
+
+// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
+// #define MI_STAT 1
+
+// Define MI_SECURE to enable security mitigations
+// #define MI_SECURE 1  // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption
+// #define MI_SECURE 2  // randomize relative allocation addresses (within mimalloc pages)
+// #define MI_SECURE 3  // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
+// #define MI_SECURE 4  // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)
+// #define MI_SECURE 5  // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)
+
+#if !defined(MI_SECURE)
+#define MI_SECURE 0
+#endif
+
+// Define MI_DEBUG for assertion and invariant checking
+// #define MI_DEBUG 1  // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON)
+// #define MI_DEBUG 2  // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON)
+// #define MI_DEBUG 3  // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON)
+#if !defined(MI_DEBUG)
+#if defined(MI_BUILD_RELEASE) || defined(NDEBUG)
+#define MI_DEBUG 0
+#else
+#define MI_DEBUG 2
+#endif
+#endif
+
+// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking)
+#ifndef MI_STAT
+#if (MI_DEBUG>0)
+#define MI_STAT 2
+#else
+#define MI_STAT 0
+#endif
+#endif
+
+// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options)
+#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL 
+#define MI_GUARDED  1
+#endif
+
+// Reserve extra padding at the end of each block to be more resilient against theap block overflows.
+// The padding can detect buffer overflow on free.
+#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))
+#define MI_PADDING  1
+#endif
+
+// Check padding bytes; allows byte-precise buffer overflow detection
+#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1)
+#define MI_PADDING_CHECK 1
+#endif
+
+
+// Encoded free lists allow detection of corrupted free lists
+// and can detect buffer overflows, modify after free, and double `free`s.
+#if (MI_SECURE>=3 || MI_DEBUG>=1)
+#define MI_ENCODE_FREELIST  1
+#endif
+
+// Enable large pages for objects between 64KiB and 512KiB.
+// This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB
+// are quite random which can lead to too many partially used large pages (but see issue #1104).
+#ifndef MI_ENABLE_LARGE_PAGES
+#define MI_ENABLE_LARGE_PAGES  1
+#endif
+
+// Place page meta info at the start of the page area or keep it separate? 
+// Separate keeps the page info at the arena start (default) which is more secure 
+// and reduces wasted space due to alignment and block sizes. 
+// (but also reserves more memory up front (about 2MiB per GiB))
+#if !defined(MI_PAGE_META_IS_SEPARATED)
+#if MI_PAGE_MAP_FLAT
+#define MI_PAGE_META_IS_SEPARATED    0
+#else
+#define MI_PAGE_META_IS_SEPARATED    1
+#endif
+#endif
+
+// We can choose to only put page info of small pages at the start of the page area.
+// This can be used to have a slightly faster `mi_free_small` function for specialized
+// cases (like language runtime systems).
+#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL)
+#define MI_PAGE_META_ALIGNED_FREE_SMALL   0
+#endif
+
+// Configuration checks
+#if !MI_PAGE_META_IS_SEPARATED && MI_SECURE
+#error "secure mode should use separated page infos"
+#endif
+#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE
+#error "secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL"
+#endif
+#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT
+#error "cannot have a flat page map with separated page infos"
+#endif
+#if MI_DEBUG && NDEBUG
+#warning "mimalloc assertions enabled in a release build"
+#endif
+
+
+// --------------------------------------------------------------
+// Sizes of internal data-structures
+// (comments specify sizes on 64-bit, usually 32-bit is halved)
+// --------------------------------------------------------------
+
+// Main size parameter; determines max arena sizes and max arena object sizes etc.
+#ifndef MI_ARENA_SLICE_SHIFT
+  #ifdef  MI_SMALL_PAGE_SHIFT   // backward compatibility
+  #define MI_ARENA_SLICE_SHIFT              MI_SMALL_PAGE_SHIFT
+  #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64
+  #define MI_ARENA_SLICE_SHIFT              (17)                        // 128 KiB to not waste too much due to 16 KiB guard pages
+  #else
+  #define MI_ARENA_SLICE_SHIFT              (13 + MI_SIZE_SHIFT)        // 64 KiB (32 KiB on 32-bit)
+  #endif
+#endif
+#if MI_ARENA_SLICE_SHIFT < 12
+#error Arena slices should be at least 4KiB
+#endif
+
+#ifndef MI_BCHUNK_BITS_SHIFT
+  #if MI_ARENA_SLICE_SHIFT <= 13    // <= 8KiB
+  #define MI_BCHUNK_BITS_SHIFT              (7)   // 128 bits
+  #elif MI_ARENA_SLICE_SHIFT < 16   // <= 32KiB
+  #define MI_BCHUNK_BITS_SHIFT              (8)   // 256 bits
+  #else
+  #define MI_BCHUNK_BITS_SHIFT              (6 + MI_SIZE_SHIFT)       // 512 bits (or 256 on 32-bit)
+  #endif
+#endif
+
+#define MI_BCHUNK_BITS                    (1 << MI_BCHUNK_BITS_SHIFT)         // sub-bitmaps in arena's are "bchunks" of 512 bits
+#define MI_ARENA_SLICE_SIZE               (MI_ZU(1) << MI_ARENA_SLICE_SHIFT)  // arena's allocate in slices of 64 KiB
+#define MI_ARENA_SLICE_ALIGN              (MI_ARENA_SLICE_SIZE)
+
+#define MI_ARENA_MIN_OBJ_SLICES           (1)
+#define MI_ARENA_MAX_CHUNK_OBJ_SLICES     (MI_BCHUNK_BITS)                    // 32 MiB (or 8 MiB on 32-bit)
+
+#define MI_ARENA_MIN_OBJ_SIZE             (MI_ARENA_MIN_OBJ_SLICES * MI_ARENA_SLICE_SIZE)
+#define MI_ARENA_MAX_CHUNK_OBJ_SIZE       (MI_ARENA_MAX_CHUNK_OBJ_SLICES * MI_ARENA_SLICE_SIZE)
+
+#if MI_ARENA_MAX_CHUNK_OBJ_SIZE < MI_SIZE_SIZE*1024
+#error maximum object size may be too small to hold local thread data
+#endif
+
+#define MI_SMALL_PAGE_SIZE                MI_ARENA_MIN_OBJ_SIZE                    // 64 KiB
+#define MI_MEDIUM_PAGE_SIZE               (8*MI_SMALL_PAGE_SIZE)                   // 512 KiB  (=byte in the bchunk bitmap)
+#define MI_LARGE_PAGE_SIZE                (MI_SIZE_SIZE*MI_MEDIUM_PAGE_SIZE)       // 4 MiB    (=word in the bchunk bitmap)
+
+
+// Maximum number of size classes. (spaced exponentially in 12.5% increments)
+#if MI_BIN_HUGE != 73U
+#error "mimalloc internal: expecting 73 bins"
+#endif
+#define MI_BIN_FULL  (MI_BIN_HUGE+1)
+#define MI_BIN_COUNT (MI_BIN_FULL+1)
+
+// We never allocate more than PTRDIFF_MAX (see also <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
+#define MI_MAX_ALLOC_SIZE        PTRDIFF_MAX
+
+// Minimal commit for a page on-demand commit (should be >= OS page size)
+#define MI_PAGE_MIN_COMMIT_SIZE  MI_ARENA_SLICE_SIZE
+
+
+// ------------------------------------------------------
+// Arena's are large reserved areas of memory allocated from
+// the OS that are managed by mimalloc to efficiently
+// allocate MI_ARENA_SLICE_SIZE slices of memory for the
+// mimalloc pages.
+// ------------------------------------------------------
+
+// A large memory arena where pages are allocated in.
+typedef struct mi_arena_s mi_arena_t;     // defined below
+
+
+// ------------------------------------------------------
+// Heaps contain allocated blocks. Heaps are self-contained
+// but share the (sub-process) memory in the arena's.
+// ------------------------------------------------------
+
+// A first-class heap.
+typedef struct mi_heap_s mi_heap_t;       // heaps
+
+// ------------------------------------------------------
+// We can have sub-processes that are fully separated
+// from each other (for running multiple Python interpreters
+// for example). A sub-process holds the memory arenas and heaps.
+// ------------------------------------------------------
+
+// A sub-process
+typedef struct mi_subproc_s mi_subproc_t;
+
+
+// ---------------------------------------------------------------
+// a memory id tracks the provenance of arena/OS allocated memory
+// ---------------------------------------------------------------
+
+// Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated.
+// The memid keeps track of this.
+typedef enum mi_memkind_e {
+  MI_MEM_NONE,      // not allocated
+  MI_MEM_EXTERNAL,  // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example)
+  MI_MEM_STATIC,    // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`))
+  MI_MEM_META,      // allocated with the meta data allocator (`arena-meta.c`)
+  MI_MEM_OS,        // allocated from the OS
+  MI_MEM_OS_HUGE,   // allocated as huge OS pages (usually 1GiB, pinned to physical memory)
+  MI_MEM_OS_REMAP,  // allocated in a remapable area (i.e. using `mremap`)
+  MI_MEM_ARENA,     // allocated from an arena (the usual case) (`arena.c`)
+  MI_MEM_HEAP_MAIN  // allocated in the main heap (for theaps)
+} mi_memkind_t;
+
+static inline bool mi_memkind_is_os(mi_memkind_t memkind) {
+  return (memkind >= MI_MEM_OS && memkind <= MI_MEM_OS_REMAP);
+}
+
+static inline bool mi_memkind_needs_no_free(mi_memkind_t memkind) {
+  return (memkind <= MI_MEM_STATIC);
+}
+
+
+typedef struct mi_memid_os_info {
+  void*         base;               // actual base address of the block (used for offset aligned allocations)
+  size_t        size;               // allocated full size
+  // size_t        alignment;       // alignment at allocation
+} mi_memid_os_info_t;
+
+typedef struct mi_memid_arena_info {
+  mi_arena_t*   arena;              // arena that contains this memory
+  uint32_t      slice_index;        // slice index in the arena
+  uint32_t      slice_count;        // allocated slices
+} mi_memid_arena_info_t;
+
+typedef struct mi_memid_meta_info {
+  void*         meta_page;          // meta-page that contains the block
+  uint32_t      block_index;        // block index in the meta-data page
+  uint32_t      block_count;        // allocated blocks
+} mi_memid_meta_info_t;
+
+typedef struct mi_memid_s {
+  union {
+    mi_memid_os_info_t    os;       // only used for MI_MEM_OS
+    mi_memid_arena_info_t arena;    // only used for MI_MEM_ARENA
+    mi_memid_meta_info_t  meta;     // only used for MI_MEM_META
+  } mem;
+  mi_memkind_t  memkind;
+  bool          is_pinned;          // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages)
+  bool          initially_committed;// `true` if the memory was originally allocated as committed
+  bool          initially_zero;     // `true` if the memory was originally zero initialized
+} mi_memid_t;
+
+
+static inline bool mi_memid_is_os(mi_memid_t memid) {
+  return mi_memkind_is_os(memid.memkind);
+}
+
+static inline bool mi_memid_needs_no_free(mi_memid_t memid) {
+  return mi_memkind_needs_no_free(memid.memkind);
+}
+
+static inline mi_arena_t* mi_memid_arena(mi_memid_t memid) {
+  return (memid.memkind == MI_MEM_ARENA ? memid.mem.arena.arena : NULL);
+}
+
+
+// ------------------------------------------------------
+// Mimalloc pages contain allocated blocks
+// ------------------------------------------------------
+
+// The free lists use encoded next fields
+// (Only actually encodes when MI_ENCODED_FREELIST is defined.)
+typedef uintptr_t  mi_encoded_t;
+
+// thread id's
+typedef size_t     mi_threadid_t;
+
+// free lists contain blocks
+typedef struct mi_block_s {
+  mi_encoded_t next;
+} mi_block_t;
+
+
+// The page flags are put in the bottom 2 bits of the thread_id (for a fast test in `mi_free`)
+// If `has_interior_pointers` is true if the page has pointers at an offset in a block (so we have to unalign to the block start before free-ing)
+// `in_full_queue` is true if the page is full and resides in the full queue (so we move it to a regular queue on free-ing)
+#define MI_PAGE_IN_FULL_QUEUE           MI_ZU(0x01)
+#define MI_PAGE_HAS_INTERIOR_POINTERS   MI_ZU(0x02)
+#define MI_PAGE_FLAG_MASK               MI_ZU(0x03)
+typedef size_t mi_page_flags_t;
+
+// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue),
+// and 4 for abandoned & mapped threads -- abandoned-mapped pages are abandoned but also mapped
+// in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse.
+// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)
+#define MI_THREADID_ABANDONED           MI_ZU(0)
+#define MI_THREADID_ABANDONED_MAPPED    (MI_PAGE_FLAG_MASK + 1)
+
+// Thread free list.
+// Points to a list of blocks that are freed by other threads.
+// The least-bit is set if the page is owned by the current thread. (`mi_page_is_owned`).
+// Ownership is required before we can read any non-atomic fields in the page.
+// This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`.
+typedef uintptr_t mi_thread_free_t;
+
+// A page contains blocks of one specific size (`block_size`).
+// Each page has three list of free blocks:
+// `free` for blocks that can be allocated,
+// `local_free` for freed blocks that are not yet available to `mi_malloc`
+// `thread_free` for freed blocks by other threads
+// The `local_free` and `thread_free` lists are migrated to the `free` list
+// when it is exhausted. The separate `local_free` list is necessary to
+// implement a monotonic heartbeat. The `thread_free` list is needed for
+// avoiding atomic operations when allocating from the owning thread.
+//
+// `used - |thread_free|` == actual blocks that are in use (alive)
+// `used - |thread_free| + |free| + |local_free| == capacity`
+//
+// We don't count "freed" (as |free|) but use only the `used` field to reduce
+// the number of memory accesses in the `mi_page_all_free` function(s).
+// Use `_mi_page_free_collect` to collect the thread_free list and update the `used` count.
+//
+// Notes:
+// - Non-atomic fields can only be accessed if having _ownership_ (low bit of `xthread_free` is 1).
+//   Combining the `thread_free` list with an ownership bit allows a concurrent `free` to atomically
+//   free an object and (re)claim ownership if the page was abandoned.
+// - If a page is not part of a theap it is called "abandoned"  (`theap==NULL`) -- in
+//   that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that
+//   are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages).
+// - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`.
+// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`
+// - Using `uint16_t` does not seem to slow things down
+
+typedef struct mi_page_s {
+  _Atomic(mi_threadid_t)    xthread_id;        // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`)
+
+  mi_block_t*               free;              // list of available free blocks (`malloc` allocates from this list)
+  uint16_t                  used;              // number of blocks in use (including blocks in `thread_free`)
+  uint16_t                  capacity;          // number of blocks committed
+  uint16_t                  reserved;          // number of blocks reserved in memory
+  uint8_t                   retire_expire;     // expiration count for retired blocks
+  bool                      free_is_zero;      // `true` if the blocks in the free list are zero initialized
+
+  mi_block_t*               local_free;        // list of deferred free blocks by this thread (migrates to `free`)
+  _Atomic(mi_thread_free_t) xthread_free;      // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`)
+
+  size_t                    block_size;        // const: size available in each block (always `>0`)
+  uint8_t*                  page_start;        // const: start of the blocks
+
+  #if (MI_ENCODE_FREELIST || MI_PADDING)
+  uintptr_t                 keys[2];           // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary
+  #endif
+
+  mi_theap_t*               theap;             // the theap owning this page (may not be valid or NULL for abandoned pages)
+  mi_heap_t*                heap;              // const: the heap owning this page
+
+  struct mi_page_s*         next;              // next page owned by the theap with the same `block_size`
+  struct mi_page_s*         prev;              // previous page owned by the theap with the same `block_size`
+  size_t                    slice_committed;   // committed size relative to the first arena slice of the page data (or 0 if the page is fully committed already)
+  mi_memid_t                memid;             // const: provenance of the page memory
+} mi_page_t;
+
+
+// ------------------------------------------------------
+// Object sizes
+// ------------------------------------------------------
+
+#define MI_PAGE_ALIGN                     MI_ARENA_SLICE_ALIGN      // pages must be aligned on this for the page map.
+#define MI_PAGE_MIN_START_BLOCK_ALIGN     MI_MAX_ALIGN_SIZE         // minimal block alignment for the first block in a page (16b)
+#define MI_PAGE_MAX_START_BLOCK_ALIGN2    (4*MI_KiB)                // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment)
+#define MI_PAGE_OSPAGE_BLOCK_ALIGN2       (4*MI_KiB)                // also aligns any multiple of this size to avoid TLB misses.
+#define MI_PAGE_MAX_OVERALLOC_ALIGN       MI_ARENA_SLICE_SIZE       // (64 KiB) limit for which we overallocate in arena pages, beyond this use OS allocation
+
+// The max object sizes are intended to not waste more than ~ 12.5% internally over the page sizes.
+#define MI_SMALL_MAX_OBJ_SIZE             ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6)   // = 10 KiB
+#if MI_ENABLE_LARGE_PAGES
+#define MI_MEDIUM_MAX_OBJ_SIZE            ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6)  // ~ 84 KiB
+#define MI_LARGE_MAX_OBJ_SIZE             (MI_LARGE_PAGE_SIZE/8)    // <= 512 KiB // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
+#else
+#define MI_MEDIUM_MAX_OBJ_SIZE            (MI_MEDIUM_PAGE_SIZE/8)   // <= 64 KiB
+#define MI_LARGE_MAX_OBJ_SIZE             MI_MEDIUM_MAX_OBJ_SIZE    // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
+#endif
+#define MI_LARGE_MAX_OBJ_WSIZE            (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE)
+
+#if (MI_LARGE_MAX_OBJ_WSIZE >= 655360)
+#error "mimalloc internal: define more bins"
+#endif
+
+
+// ------------------------------------------------------
+// Page kinds
+// ------------------------------------------------------
+
+typedef enum mi_page_kind_e {
+  MI_PAGE_SMALL,      // small blocks go into 64KiB pages
+  MI_PAGE_MEDIUM,     // medium blocks go into 512KiB pages
+  MI_PAGE_LARGE,      // larger blocks go into 4MiB pages (if `MI_ENABLE_LARGE_PAGES==1`)
+  MI_PAGE_SINGLETON   // page containing a single block.
+                      // used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an alignment `> MI_PAGE_MAX_OVERALLOC_ALIGN`.
+} mi_page_kind_t;
+
+
+
+// ------------------------------------------------------
+// A "theap" is a thread local heap which owns pages.
+// (making them thread-local avoids atomic operations)
+//
+// All theaps belong to a (non-thread-local) heap.
+// A theap just owns a set of pages for allocation and
+// can only be allocate/reallocate from the thread that created it.
+// Freeing blocks can be done from any thread though.
+//
+// Per thread, there is always a default theap that belongs
+// to the default heap. It is initialized to statically
+// point initially to an empty theap to avoid initialization
+// checks in the fast path.
+// ------------------------------------------------------
+
+// Thread local data
+typedef struct mi_tld_s mi_tld_t;   // defined below
+
+// Pages of a certain block size are held in a queue.
+typedef struct mi_page_queue_s {
+  mi_page_t* first;
+  mi_page_t* last;
+  size_t     count;
+  size_t     block_size;
+} mi_page_queue_t;
+
+// Random context
+typedef struct mi_random_cxt_s {
+  uint32_t input[16];
+  uint32_t output[16];
+  int      output_available;
+  bool     weak;
+} mi_random_ctx_t;
+
+
+// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows
+#if MI_PADDING
+typedef struct mi_padding_s {
+  uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
+  uint32_t delta;  // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
+} mi_padding_t;
+#define MI_PADDING_SIZE   (sizeof(mi_padding_t))
+#define MI_PADDING_WSIZE  ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE)
+#else
+#define MI_PADDING_SIZE   0
+#define MI_PADDING_WSIZE  0
+#endif
+
+#define MI_PAGES_DIRECT   (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)
+
+
+// A thread-local heap ("theap") owns a set of thread-local pages.
+struct mi_theap_s {
+  mi_tld_t*             tld;                                 // thread-local data
+  _Atomic(mi_heap_t*)   heap;                                // the heap this theap belongs to.
+  _Atomic(size_t)       refcount;                            // reference count
+  unsigned long long    heartbeat;                           // monotonic heartbeat count
+  uintptr_t             cookie;                              // random cookie to verify pointers (see `_mi_ptr_cookie`)
+  mi_random_ctx_t       random;                              // random number context used for secure allocation
+  size_t                page_count;                          // total number of pages in the `pages` queues.
+  size_t                page_retired_min;                    // smallest retired index (retired pages are fully free, but still in the page queues)
+  size_t                page_retired_max;                    // largest retired index into the `pages` array.
+  size_t                pages_full_size;                     // optimization: total size of blocks in the pages of the full queue (issue #1220)
+  long                  generic_count;                       // how often is `_mi_malloc_generic` called?
+  long                  generic_collect_count;               // how often is `_mi_malloc_generic` called without collecting?
+
+  mi_theap_t*           tnext;                               // list of theaps in this thread
+  mi_theap_t*           tprev;  
+  mi_theap_t*           hnext;                               // list of theaps of the owning `heap`
+  mi_theap_t*           hprev;
+  
+  long                  page_full_retain;                    // how many full pages can be retained per queue (before abandoning them)
+  bool                  allow_page_reclaim;                  // `true` if this theap should not reclaim abandoned pages
+  bool                  allow_page_abandon;                  // `true` if this theap can abandon pages to reduce memory footprint
+  #if MI_GUARDED
+  size_t                guarded_size_min;                    // minimal size for guarded objects
+  size_t                guarded_size_max;                    // maximal size for guarded objects
+  size_t                guarded_sample_rate;                 // sample rate (set to 0 to disable guarded pages)
+  size_t                guarded_sample_count;                // current sample count (counting down to 0)
+  #endif
+  mi_page_t*            pages_free_direct[MI_PAGES_DIRECT];  // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
+  mi_page_queue_t       pages[MI_BIN_COUNT];                 // queue of pages for each size class (or "bin")
+  mi_memid_t            memid;                               // provenance of the theap struct itself (meta or os)
+  mi_stats_t            stats;                               // thread-local statistics
+};
+
+
+
+
+// ------------------------------------------------------
+// Heaps contain allocated blocks. Heaps are self-contained
+// but share the (sub-process) memory in the arena's.
+// ------------------------------------------------------
+
+// Keep track of all owned and abandoned pages in the arena's
+struct mi_arena_pages_s;
+typedef struct mi_arena_pages_s mi_arena_pages_t;
+
+#define MI_MAX_ARENAS   (160)   // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `mi_arena_reserve`)
+                                // 160 arenas is enough for ~2 TiB memory
+
+// A dynamic thread-local variable; 0 for an invalid thread-local
+typedef size_t mi_thread_local_t;
+
+typedef struct mi_heap_s {
+  mi_subproc_t*         subproc;                        // a heap belongs to a subprocess
+  size_t                heap_seq;                       // unique sequence number for heaps in this subprocess
+  mi_heap_t*            next;                           // list of heaps in this subprocess
+  mi_heap_t*            prev;
+  mi_thread_local_t     theap;                          // dynamic thread local for the thread-local theaps of this heap
+
+  mi_arena_t*           exclusive_arena;                // if the heap should only allocate from a specific arena (or NULL)
+  int                   numa_node;                      // if >=0, prefer this numa node for allocations
+
+  mi_theap_t*           theaps;                         // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields)
+  mi_lock_t             theaps_lock;                    // lock for the theaps list operations
+
+  _Atomic(size_t)       abandoned_count[MI_BIN_COUNT];  // total count of abandoned pages in this heap
+  mi_page_t*            os_abandoned_pages;             // list of pages that are OS allocated and not in an arena
+  mi_lock_t             os_abandoned_pages_lock;        // lock for the os abandoned pages list (this lock protects list operations)
+
+  _Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL)
+  mi_lock_t             arena_pages_lock;                // lock to update the arena_pages array
+
+  mi_stats_t            stats;                           // statistics for this heap; periodically updated by merging from each theap
+} mi_heap_t;
+
+
+// ------------------------------------------------------
+// Sub processes do not reclaim or visit pages from other sub processes.
+// These are essentially the static variables of a process, and
+// usually there is only one subprocess. This can be used for example
+// by CPython to have separate interpreters within one process.
+// Each thread can only belong to one subprocess
+// (and needs to call `mi_subproc_add_current_thread` before any allocations).
+// ------------------------------------------------------
+
+struct mi_subproc_s {
+  size_t                subproc_seq;                    // unique id for sub-processes
+  mi_subproc_t*         next;                           // list of all sub-processes
+  mi_subproc_t*         prev;
+
+  _Atomic(size_t)       arena_count;                    // current count of arena's
+  _Atomic(mi_arena_t*)  arenas[MI_MAX_ARENAS];          // arena's of this sub-process
+  mi_lock_t             arena_reserve_lock;             // lock to ensure arena's get reserved one at a time
+  mi_decl_align(8)                                      // needed on some 32-bit platforms
+  _Atomic(int64_t)      purge_expire;                   // expiration is set if any arenas can be purged
+
+  _Atomic(mi_heap_t*)   heap_main;                      // main heap for this sub process
+  mi_heap_t*            heaps;                          // heaps belonging to this sub-process
+  mi_lock_t             heaps_lock;
+
+  _Atomic(size_t)       thread_count;                   // current threads associated with this sub-process
+  _Atomic(size_t)       thread_total_count;             // total created threads associated with this sub-process
+  _Atomic(size_t)       heap_count;                     // current heaps in this sub-process (== |heaps|)
+  _Atomic(size_t)       heap_total_count;               // total created heaps in this sub-process
+
+  mi_memid_t            memid;                          // provenance of this memory block (meta or static)
+  mi_decl_align(8)                                      // needed on some 32-bit platforms
+  mi_stats_t            stats;                          // subprocess statistics; updated for arena/OS stats like committed,
+                                                        // and otherwise merged with heap stats when those are deleted
+};
+
+
+// ------------------------------------------------------
+// Thread Local data
+// ------------------------------------------------------
+
+// Milliseconds as in `int64_t` to avoid overflows
+typedef int64_t  mi_msecs_t;
+
+// Thread local data
+struct mi_tld_s {
+  mi_threadid_t         thread_id;            // thread id of this thread
+  size_t                thread_seq;           // thread sequence id (linear count of created threads)
+  int                   numa_node;            // thread preferred numa node
+  mi_subproc_t*         subproc;              // sub-process this thread belongs to.
+  mi_theap_t*           theaps;               // list of theaps in this thread (so we can abandon all when the thread terminates)
+  mi_lock_t             theaps_lock;          // lock as the theaps list is sometimes accessed from another thread (on `mi_heap_free`)
+  bool                  recurse;              // true if deferred was called; used to prevent infinite recursion.
+  bool                  is_in_threadpool;     // true if this thread is part of a threadpool (and can run arbitrary tasks)
+  mi_memid_t            memid;                // provenance of the tld memory itself (meta or OS)
+};
+
+
+/* ----------------------------------------------------------------------------
+  Arenas are fixed area's of OS memory from which we can allocate
+  large blocks (>= MI_ARENA_MIN_BLOCK_SIZE).
+  In contrast to the rest of mimalloc, the arenas are shared between
+  threads and need to be accessed using atomic operations (using atomic `mi_bitmap_t`'s).
+
+  Arenas are also used to for huge OS page (1GiB) reservations or for reserving
+  OS memory upfront which can be improve performance or is sometimes needed
+  on embedded devices. We can also employ this with WASI or `sbrk` systems
+  to reserve large arenas upfront and be able to reuse the memory more effectively.
+-----------------------------------------------------------------------------*/
+
+#define MI_ARENA_BIN_COUNT      (MI_BIN_COUNT)
+#define MI_ARENA_MIN_SIZE       (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE)           // 32 MiB (or 8 MiB on 32-bit)
+#define MI_ARENA_MAX_SIZE       (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE)
+
+typedef struct mi_bitmap_s  mi_bitmap_t;    // atomic bitmap  (defined in `src/bitmap.h`)
+typedef struct mi_bbitmap_s mi_bbitmap_t;   // atomic binned bitmap (defined in `src/bitmap.h`)
+
+typedef struct mi_arena_pages_s {
+  mi_bitmap_t* pages;                // all registered pages (abandoned and owned)
+  mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT];  // abandoned pages per size bin (a set bit means the start of the page)
+  // followed by the bitmaps (whose siz`es depend on the arena size)
+} mi_arena_pages_t;
+
+
+// A memory arena
+typedef struct mi_arena_s {
+  mi_memid_t          memid;                // provenance of the memory area
+  mi_subproc_t*       subproc;              // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`)
+  size_t              arena_idx;            // index in the arenas array
+
+  size_t              slice_count;          // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)
+  size_t              info_slices;          // initial slices reserved for the arena bitmaps
+  int                 numa_node;            // associated NUMA node
+  bool                is_exclusive;         // only allow allocations if specifically for this arena
+  mi_decl_align(8)                          // needed on some 32-bit platforms
+  _Atomic(mi_msecs_t) purge_expire;         // expiration time when slices can be purged from `slices_purge`.
+  mi_commit_fun_t*    commit_fun;           // custom commit/decommit memory
+  void*               commit_fun_arg;       // user argument for a custom commit function
+
+  size_t              total_size;           // for (user given) memory more than MI_ARENA_MAX_SIZE, we use N arena's to cover it. The first (parent) has the total size (and the other sub-arena's 0).
+  mi_arena_t*         parent;               // if this is a sub arena, this points to the first one in the memory area.
+
+  mi_bbitmap_t*       slices_free;          // is the slice free? (a binned bitmap with size classes)
+  mi_bitmap_t*        slices_committed;     // is the slice committed? (i.e. accessible)
+  mi_bitmap_t*        slices_dirty;         // is the slice potentially non-zero?
+  mi_bitmap_t*        slices_purge;         // slices that can be purged
+  mi_page_t*          pages_meta;           // pre-allocated `slice_count` page meta info -- only used if `MI_PAGE_META_IS_SEPARATED!=0`
+  mi_arena_pages_t    pages_main;           // arena page bitmaps for the main heap are allocated up front as well
+
+  // followed by the bitmaps (whose sizes depend on the arena size)
+  // note: when adding bitmaps revise `mi_arena_info_slices_needed`
+} mi_arena_t;
+
+
+
+/* -----------------------------------------------------------
+  Error codes passed to `_mi_fatal_error`
+  All are recoverable but EFAULT is a serious error and aborts by default in secure mode.
+  For portability define undefined error codes using common Unix codes:
+  <https://www-numi.fnal.gov/offline_software/srt_public_context/WebDocs/Errors/unix_system_errors.html>
+----------------------------------------------------------- */
+
+#ifndef EAGAIN         // double free
+#define EAGAIN (11)
+#endif
+#ifndef ENOMEM         // out of memory
+#define ENOMEM (12)
+#endif
+#ifndef EFAULT         // corrupted free-list or meta-data
+#define EFAULT (14)
+#endif
+#ifndef EINVAL         // trying to free an invalid pointer
+#define EINVAL (22)
+#endif
+#ifndef EOVERFLOW      // count*size overflow
+#define EOVERFLOW (75)
+#endif
+
+/* -----------------------------------------------------------
+  Debug constants
+----------------------------------------------------------- */
+
+#if !defined(MI_DEBUG_UNINIT)
+#define MI_DEBUG_UNINIT     (0xD0)
+#endif
+#if !defined(MI_DEBUG_FREED)
+#define MI_DEBUG_FREED      (0xDF)
+#endif
+#if !defined(MI_DEBUG_PADDING)
+#define MI_DEBUG_PADDING    (0xDE)
+#endif
+
+
+#endif // MI_TYPES_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-aligned.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-aligned.c
new file mode 100644
index 00000000..97dc6045
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-aligned.c
@@ -0,0 +1,441 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"  // _mi_theap_default
+
+#include <string.h>     // memset
+
+// ------------------------------------------------------
+// Aligned Allocation
+// ------------------------------------------------------
+
+static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {
+  // certain blocks are always allocated at a certain natural alignment.
+  // (see also `arena.c:mi_arenas_page_alloc_fresh`).
+  mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0));
+  if (alignment > size) return false;
+  const size_t bsize = mi_good_size(size);
+  const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) ||             // power-of-two under N
+                  (alignment==MI_PAGE_OSPAGE_BLOCK_ALIGN2 && (bsize % MI_PAGE_OSPAGE_BLOCK_ALIGN2)==0);  // or multiple of N
+  if (ok) { mi_assert_internal((bsize & (alignment-1)) == 0); } // since both power of 2 and alignment <= size
+  return ok;
+}
+
+#if MI_GUARDED
+static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero) mi_attr_noexcept {
+  // use over allocation for guarded blocksl
+  #if MI_THEAP_INITASNULL
+  if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }
+  #endif
+  mi_assert_internal(alignment > 0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN);
+  if mi_unlikely(alignment >= MI_PAGE_MAX_OVERALLOC_ALIGN || size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE - alignment)) {
+    _mi_error_message(EOVERFLOW, "(guarded) aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
+    return NULL;
+  }
+  const size_t oversize = size + alignment - 1;
+  void* const base = _mi_theap_malloc_guarded(theap, oversize, zero);
+  if (base==NULL) return NULL;
+  void* const p = _mi_align_up_ptr(base, alignment);
+  mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size);
+  mi_assert_internal(mi_usable_size(p) >= size);
+  mi_assert_internal(_mi_is_aligned(p, alignment));
+  return p;
+}
+
+static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {
+  #if MI_THEAP_INITASNULL
+  if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }
+  #endif
+  const size_t rate = theap->guarded_sample_rate;
+  // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty`
+  if (rate != 0) { theap->guarded_sample_rate = 0; }
+  void* p = _mi_theap_malloc_zero(theap, size, zero, usable);
+  if (rate != 0) { theap->guarded_sample_rate = rate; }
+  return p;
+}
+#else
+static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {
+  return _mi_theap_malloc_zero(theap, size, zero, usable);
+}
+#endif
+
+// Fallback aligned allocation that over-allocates -- split out for better codegen
+static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
+{
+  mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE));
+  mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
+
+  void* p;
+  size_t oversize;
+  if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
+    // use OS allocation for large alignments and allocate inside a singleton page (not in an arena)
+    // This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned
+    // in the first (and single) page such that the page info is `MI_PAGE_ALIGN` bytes before it (and can be found in the _mi_page_map).
+    if mi_unlikely(offset != 0) {
+      // todo: cannot support offset alignment for very large alignments yet
+      _mi_error_message(EOVERFLOW, "aligned allocation with a large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset);
+      return NULL;
+    }
+    oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size);
+    // note: no guarded as alignment > 0
+    p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, usable); // the page block size should be large enough to align in the single huge page block
+    if (p == NULL) return NULL;
+  }
+  else {
+    // otherwise over-allocate
+    mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN);
+    mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow
+    oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1;  // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that.
+    p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, usable);
+    if (p == NULL) return NULL;
+  }
+
+  // .. and align within the allocation
+  const uintptr_t align_mask = alignment - 1;  // for any x, `(x & align_mask) == (x % alignment)`
+  const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask;
+  const uintptr_t adjust  = (poffset == 0 ? 0 : alignment - poffset);
+  mi_assert_internal(adjust < alignment);
+  void* aligned_p = (void*)((uintptr_t)p + adjust);
+
+  // note: after the above allocation, the page may be abandoned now (as it became full, see `page.c:_mi_malloc_generic`)
+  // and we no longer own it. We should be careful to only read constant fields in the page,
+  // or use safe atomic access as in `mi_page_set_has_interior_pointers`.
+  // (we can access the page though since the just allocated pointer keeps it alive)
+  mi_page_t* page = _mi_ptr_page(p);
+  if (aligned_p != p) {
+    mi_page_set_has_interior_pointers(page, true);
+    #if MI_GUARDED
+    // set tag to aligned so mi_usable_size works with guard pages
+    if (adjust >= sizeof(mi_block_t)) {
+      mi_block_t* const block = (mi_block_t*)p;
+      block->next = MI_BLOCK_TAG_ALIGNED;
+    }
+    #endif
+    _mi_padding_shrink(page, (mi_block_t*)p, adjust + size);
+  }
+  // todo: expand padding if overallocated ?
+
+  mi_assert_internal(mi_page_usable_block_size(page) >= adjust + size);
+  mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);
+  mi_assert_internal(mi_usable_size(aligned_p)>=size);
+  mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust);
+  #if MI_DEBUG > 1
+  mi_page_t* const apage = _mi_ptr_page(aligned_p);
+  void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);
+  mi_assert_internal(p == unalign_p);
+  #endif
+
+  // now zero the block if needed
+  //if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
+  //  // for the tracker, on huge aligned allocations only from the start of the large block is defined
+  //  mi_track_mem_undefined(aligned_p, size);
+  //  if (zero) {
+  //    _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p));
+  //  }
+  //}
+
+  if (p != aligned_p) {
+    mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p));
+    #if MI_GUARDED
+    mi_track_mem_defined(p, sizeof(mi_block_t));
+    #endif
+  }
+  return aligned_p;
+}
+
+// Generic primitive aligned allocation -- split out for better codegen
+static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
+{
+  mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
+  // we don't allocate more than MI_MAX_ALLOC_SIZE (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
+  if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {
+    _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
+    return NULL;
+  }
+
+  // use regular allocation if it is guaranteed to fit the alignment constraints.
+  // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist
+  // a page with the right block size, and if we always use the over-alloc fallback that would never happen.
+  if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) {
+    void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, usable);
+    mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
+    const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;
+    if mi_likely(is_aligned_or_null) {
+      return p;
+    }
+    else {
+      // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..
+      mi_assert(false);
+      mi_free(p);
+    }
+  }
+
+  // fall back to over-allocation
+  return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,usable);
+}
+
+
+// Primitive aligned allocation
+static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
+{
+  // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size.
+  if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
+    #if MI_DEBUG > 0
+    _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment);
+    #endif
+    return NULL;
+  }
+
+  #if MI_GUARDED
+  #if MI_THEAP_INITASNULL
+  if mi_likely(theap!=NULL)
+  #endif
+  if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) {
+    return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero);
+  }
+  #endif
+
+  // try first if there happens to be a small block available with just the right alignment
+  // since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already
+  // naturally aligned this can be often the case.
+  #if MI_THEAP_INITASNULL
+  if mi_likely(theap!=NULL)
+  #endif
+  {
+    if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) {
+      const uintptr_t align_mask = alignment-1;       // for any x, `(x & align_mask) == (x % alignment)`
+      const size_t padsize = size + MI_PADDING_SIZE;
+      mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize);
+      if mi_likely(page->free != NULL) {
+        const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;
+        if mi_likely(is_aligned)
+        {
+          if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
+          void* p = _mi_page_malloc_zero(theap, page, padsize, zero);
+          mi_assert_internal(p != NULL);
+          mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
+          mi_track_malloc(p, size, zero);
+          return p;
+        }
+      }
+    }
+  }
+
+  // fallback to generic aligned allocation
+  return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, usable);
+}
+
+
+// ------------------------------------------------------
+// Internal mi_theap_malloc_aligned / mi_malloc_aligned
+// ------------------------------------------------------
+
+static mi_decl_restrict void* mi_theap_malloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, false, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_malloc_aligned_at(theap, size, alignment, 0);
+}
+
+static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL);
+}
+
+static mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_zalloc_aligned_at(theap, size, alignment, 0);
+}
+
+static mi_decl_restrict void* mi_theap_calloc_aligned_at(mi_theap_t* theap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(count, size, &total)) return NULL;
+  return mi_theap_zalloc_aligned_at(theap, total, alignment, offset);
+}
+
+static mi_decl_restrict void* mi_theap_calloc_aligned(mi_theap_t* theap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_calloc_aligned_at(theap, count, size, alignment, 0);
+}
+
+
+// ------------------------------------------------------
+// Aligned Allocation
+// ------------------------------------------------------
+
+mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_malloc_aligned_at(_mi_theap_default(), size, alignment, offset);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_malloc_aligned(_mi_theap_default(), size, alignment);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {
+  return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, block_size);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_zalloc_aligned_at(_mi_theap_default(), size, alignment, offset);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_zalloc_aligned(_mi_theap_default(), size, alignment);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {
+  return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, block_size);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_calloc_aligned_at(_mi_theap_default(), count, size, alignment, offset);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_calloc_aligned(_mi_theap_default(), count, size, alignment);
+}
+
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_malloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_malloc_aligned(_mi_heap_theap(heap), size, alignment);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_zalloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_zalloc_aligned(_mi_heap_theap(heap), size, alignment);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_calloc_aligned_at(_mi_heap_theap(heap), count, size, alignment, offset);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_calloc_aligned(_mi_heap_theap(heap), count, size, alignment);
+}
+
+
+// ------------------------------------------------------
+// Aligned re-allocation
+// ------------------------------------------------------
+
+static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {
+  mi_assert(alignment > 0);
+  if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
+  if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL);
+  size_t size = mi_usable_size(p);
+  if (newsize <= size && newsize >= (size - (size / 2))
+      && (((uintptr_t)p + offset) % alignment) == 0) {
+    return p;  // reallocation still fits, is aligned and not more than 25% waste
+  }
+  else {
+    // note: we don't zero allocate upfront so we only zero initialize the expanded part
+    void* newp = mi_theap_malloc_aligned_at(theap,newsize,alignment,offset);
+    if (newp != NULL) {
+      if (zero && newsize > size) {
+        // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
+        size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
+        _mi_memzero((uint8_t*)newp + start, newsize - start);
+      }
+      _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
+      mi_free(p); // only free if successful
+    }
+    return newp;
+  }
+}
+
+static void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept {
+  mi_assert(alignment > 0);
+  if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
+  return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero);
+}
+
+static void* mi_theap_realloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,offset,false);
+}
+
+static void* mi_theap_realloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
+  return mi_theap_realloc_zero_aligned(theap,p,newsize,alignment,false);
+}
+
+static void* mi_theap_rezalloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_realloc_zero_aligned_at(theap, p, newsize, alignment, offset, true);
+}
+
+static void* mi_theap_rezalloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
+  return mi_theap_realloc_zero_aligned(theap, p, newsize, alignment, true);
+}
+
+static void* mi_theap_recalloc_aligned_at(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(newcount, size, &total)) return NULL;
+  return mi_theap_rezalloc_aligned_at(theap, p, total, alignment, offset);
+}
+
+static void* mi_theap_recalloc_aligned(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(newcount, size, &total)) return NULL;
+  return mi_theap_rezalloc_aligned(theap, p, total, alignment);
+}
+
+
+mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_realloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset);
+}
+
+mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
+  return mi_theap_realloc_aligned(_mi_theap_default(), p, newsize, alignment);
+}
+
+mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_rezalloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset);
+}
+
+mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
+  return mi_theap_rezalloc_aligned(_mi_theap_default(), p, newsize, alignment);
+}
+
+mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_recalloc_aligned_at(_mi_theap_default(), p, newcount, size, alignment, offset);
+}
+
+mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_recalloc_aligned(_mi_theap_default(), p, newcount, size, alignment);
+}
+
+
+mi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_realloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset);
+}
+
+mi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
+  return mi_theap_realloc_aligned(_mi_heap_theap(heap), p, newsize, alignment);
+}
+
+mi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_rezalloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset);
+}
+
+mi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
+  return mi_theap_rezalloc_aligned(_mi_heap_theap(heap), p, newsize, alignment);
+}
+
+mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
+  return mi_theap_recalloc_aligned_at(_mi_heap_theap(heap), p, newcount, size, alignment, offset);
+}
+
+mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
+  return mi_theap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment);
+}
+
+
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-override.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-override.c
new file mode 100644
index 00000000..93d066dc
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-override.c
@@ -0,0 +1,396 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+#if !defined(MI_IN_ALLOC_C)
+#error "this file should be included from 'alloc.c' (so aliases can work)"
+#endif
+
+
+#if defined(MI_MALLOC_OVERRIDE) && !defined(_DLL)
+
+#if defined(__APPLE__)
+#include <AvailabilityMacros.h>
+mi_decl_externc void   vfree(void* p);
+mi_decl_externc size_t malloc_size(const void* p);
+mi_decl_externc size_t malloc_good_size(size_t size);
+#endif
+
+// helper definition for C override of C++ new
+typedef void* mi_nothrow_t;
+
+// ------------------------------------------------------
+// Override system malloc
+// ------------------------------------------------------
+
+#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !MI_TRACK_ENABLED
+  // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions
+  #if (defined(__GNUC__) && __GNUC__ >= 9)
+    #pragma GCC diagnostic ignored "-Wattributes"  // or we get warnings that nodiscard is ignored on a forward
+    #define MI_FORWARD(fun)      __attribute__((alias(#fun), used, visibility("default"), copy(fun)));
+  #else
+    #define MI_FORWARD(fun)      __attribute__((alias(#fun), used, visibility("default")));
+  #endif
+  #define MI_FORWARD1(fun,x)      MI_FORWARD(fun)
+  #define MI_FORWARD2(fun,x,y)    MI_FORWARD(fun)
+  #define MI_FORWARD3(fun,x,y,z)  MI_FORWARD(fun)
+  #define MI_FORWARD0(fun,x)      MI_FORWARD(fun)
+  #define MI_FORWARD02(fun,x,y)   MI_FORWARD(fun)
+#else
+  // otherwise use forwarding by calling our `mi_` function
+  #define MI_FORWARD1(fun,x)      { return fun(x); }
+  #define MI_FORWARD2(fun,x,y)    { return fun(x,y); }
+  #define MI_FORWARD3(fun,x,y,z)  { return fun(x,y,z); }
+  #define MI_FORWARD0(fun,x)      { fun(x); }
+  #define MI_FORWARD02(fun,x,y)   { fun(x,y); }
+#endif
+
+#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE)
+  // define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for
+  // functions that are interposed (or the interposing does not work)
+  #define MI_OSX_IS_INTERPOSED
+
+  mi_decl_externc size_t mi_malloc_size_checked(void *p) {
+    if (!mi_is_in_heap_region(p)) return 0;
+    return mi_usable_size(p);
+  }
+
+  // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`
+  // See: <https://books.google.com/books?id=K8vUkpOXhN4C&pg=PA73>
+  struct mi_interpose_s {
+    const void* replacement;
+    const void* target;
+  };
+  #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }
+  #define MI_INTERPOSE_MI(fun)            MI_INTERPOSE_FUN(fun,mi_##fun)
+
+  #define MI_INTERPOSE_DECLS(name)        __attribute__((used)) static struct mi_interpose_s name[]  __attribute__((section("__DATA, __interpose")))
+
+  MI_INTERPOSE_DECLS(_mi_interposes) =
+  {
+    MI_INTERPOSE_MI(malloc),
+    MI_INTERPOSE_MI(calloc),
+    MI_INTERPOSE_MI(realloc),
+    MI_INTERPOSE_MI(strdup),
+    MI_INTERPOSE_MI(realpath),
+    MI_INTERPOSE_MI(posix_memalign),
+    MI_INTERPOSE_MI(reallocf),
+    MI_INTERPOSE_MI(valloc),
+    MI_INTERPOSE_FUN(malloc_size,mi_malloc_size_checked),
+    MI_INTERPOSE_MI(malloc_good_size),
+    #ifdef MI_OSX_ZONE
+    // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely
+    MI_INTERPOSE_MI(free),
+    MI_INTERPOSE_FUN(vfree,mi_free),
+    #else
+    // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity <https://github.com/nneonneo/osx-10.9-opensource/blob/master/objc4-551.1/runtime/hashtable2.mm>)
+    MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us
+    MI_INTERPOSE_FUN(vfree,mi_cfree),
+    #endif
+  };
+  #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
+  MI_INTERPOSE_DECLS(_mi_interposes_10_7)  = { MI_INTERPOSE_MI(strndup) };
+  #endif
+  #if defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15)
+  MI_INTERPOSE_DECLS(_mi_interposes_10_15) = { MI_INTERPOSE_MI(aligned_alloc) };
+  #endif
+
+  #ifdef __cplusplus
+  extern "C" {
+  #endif
+  void  _ZdlPv(void* p);   // delete
+  void  _ZdaPv(void* p);   // delete[]
+  void  _ZdlPvm(void* p, size_t n);  // delete
+  void  _ZdaPvm(void* p, size_t n);  // delete[]
+  void* _Znwm(size_t n);  // new
+  void* _Znam(size_t n);  // new[]
+  void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow
+  void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow
+  #ifdef __cplusplus
+  }
+  #endif
+  __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[]  __attribute__((section("__DATA, __interpose"))) =
+  {
+    MI_INTERPOSE_FUN(_ZdlPv,mi_free),
+    MI_INTERPOSE_FUN(_ZdaPv,mi_free),
+    MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size),
+    MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size),
+    MI_INTERPOSE_FUN(_Znwm,mi_new),
+    MI_INTERPOSE_FUN(_Znam,mi_new),
+    MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow),
+    MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow),
+  };
+
+#elif defined(_MSC_VER)
+  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE
+  void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
+    return mi_expand(_Block, _Size);
+  }
+  _Check_return_ _ACRTIMP 
+  size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT {
+    return mi_malloc_size(_Block);
+  }
+  _Check_return_ _ACRTIMP _CRT_HYBRIDPATCHABLE 
+  size_t __cdecl _msize(_Pre_notnull_ void* _Block) {
+    return mi_malloc_size(_Block);
+  }
+  _ACRTIMP 
+  void __cdecl _free_base(_Pre_maybenull_ _Post_invalid_ void* _Block) {
+    mi_free(_Block);
+  }
+  _ACRTIMP _CRT_HYBRIDPATCHABLE 
+  void __cdecl free(_Pre_maybenull_ _Post_invalid_ void* _Block) {
+    mi_free(_Block);
+  }
+  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_JIT_INTRINSIC _CRTRESTRICT _CRT_HYBRIDPATCHABLE
+  void* __cdecl malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size) {
+    return mi_malloc(_Size);
+  }
+  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _malloc_base(_In_ size_t _Size) {
+    return mi_malloc(_Size);
+  }
+  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _realloc_base(_Pre_maybenull_ _Post_invalid_  void*  _Block, _In_ size_t _Size) {
+    return mi_realloc(_Block, _Size);
+  }
+  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT _CRT_HYBRIDPATCHABLE
+  void* __cdecl realloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW  size_t _Size) {
+    return mi_realloc(_Block, _Size);
+  }
+  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT 
+  void* __cdecl _calloc_base(_In_ size_t _Count, _In_ size_t _Size) {
+    return mi_calloc(_Count, _Size);
+  }
+  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRT_JIT_INTRINSIC _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl calloc(_In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
+    return mi_calloc(_Count, _Size);
+  }
+  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _recalloc_base(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ size_t _Count, _In_ size_t _Size) {
+    return mi_recalloc(_Block, _Count, _Size);
+  }
+  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _recalloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
+    return mi_recalloc(_Block, _Count, _Size);
+  }
+  _ACRTIMP 
+  void __cdecl _aligned_free(_Pre_maybenull_ _Post_invalid_ void* _Block) {
+    mi_free(_Block);
+  }
+  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _aligned_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {
+    return mi_malloc_aligned(_Size, _Alignment);
+  }
+  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _aligned_offset_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {
+    return mi_malloc_aligned_at(_Size, _Alignment, _Offset);
+  }
+  _Check_return_ _ACRTIMP 
+  size_t __cdecl _aligned_msize(_Pre_notnull_ void*  _Block, _In_ size_t _Alignment, _In_ size_t _Offset) {
+    MI_UNUSED(_Alignment); MI_UNUSED(_Offset); return mi_malloc_size(_Block);
+  }
+  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _aligned_offset_realloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {
+    return mi_realloc_aligned_at(_Block, _Size, _Alignment, _Offset);
+  }
+  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _aligned_offset_recalloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {
+    return mi_recalloc_aligned_at(_Block, _Count, _Size, _Alignment, _Offset);
+  }
+  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _aligned_realloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {
+    return mi_realloc_aligned(_Block, _Size, _Alignment);
+  }
+  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
+  void* __cdecl _aligned_recalloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {
+    return mi_recalloc_aligned(_Block, _Count, _Size, _Alignment);
+  }
+#else
+  // On all other systems forward allocation primitives to our API
+  mi_decl_export void* malloc(size_t size)              MI_FORWARD1(mi_malloc, size)
+  mi_decl_export void* calloc(size_t size, size_t n)    MI_FORWARD2(mi_calloc, size, n)
+  mi_decl_export void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize)
+  mi_decl_export void  free(void* p)                    MI_FORWARD0(mi_free, p)
+  // In principle we do not need to forward `strdup`/`strndup` but on some systems these do not use `malloc` internally (but a more primitive call)
+  // We only override if `strdup` is not a macro (as on some older libc's, see issue #885)
+  #if !defined(strdup)
+  mi_decl_export char* strdup(const char* str)             MI_FORWARD1(mi_strdup, str)
+  #endif
+  #if !defined(strndup) && (!defined(__APPLE__) || (defined(MAC_OS_X_VERSION_10_7) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7))
+  mi_decl_export char* strndup(const char* str, size_t n)  MI_FORWARD2(mi_strndup, str, n)
+  #endif
+#endif
+
+#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
+#pragma GCC visibility push(default)
+#endif
+
+// ------------------------------------------------------
+// Override new/delete
+// This is not really necessary as they usually call
+// malloc/free anyway, but it improves performance.
+// ------------------------------------------------------
+#ifdef __cplusplus
+  // ------------------------------------------------------
+  // With a C++ compiler we override the new/delete operators.
+  // see <https://en.cppreference.com/w/cpp/memory/new/operator_new>
+  // ------------------------------------------------------
+  #include <new>
+
+  #ifndef MI_OSX_IS_INTERPOSED
+    void operator delete(void* p) noexcept              MI_FORWARD0(mi_free,p)
+    void operator delete[](void* p) noexcept            MI_FORWARD0(mi_free,p)
+
+    void* operator new(std::size_t n) noexcept(false)   MI_FORWARD1(mi_new,n)
+    void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)
+
+    void* operator new  (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }
+    void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }
+
+    #if (__cplusplus >= 201402L || _MSC_VER >= 1916)
+    void operator delete  (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)
+    void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)
+    #endif
+  #endif
+
+  #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5))
+  void operator delete  (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  void operator delete  (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
+  void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
+  void operator delete  (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+
+  void* operator new( std::size_t n, std::align_val_t al)   noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
+  void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
+  void* operator new  (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
+  void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
+  #endif
+
+#elif (defined(__GNUC__) || defined(__clang__))
+  // ------------------------------------------------------
+  // Override by defining the mangled C++ names of the operators (as
+  // used by GCC and CLang).
+  // See <https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling>
+  // ------------------------------------------------------
+
+  void _ZdlPv(void* p)            MI_FORWARD0(mi_free,p) // delete
+  void _ZdaPv(void* p)            MI_FORWARD0(mi_free,p) // delete[]
+  void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)
+  void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)
+
+  void _ZdlPvSt11align_val_t(void* p, size_t al)            { mi_free_aligned(p,al); }
+  void _ZdaPvSt11align_val_t(void* p, size_t al)            { mi_free_aligned(p,al); }
+  void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
+  void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
+
+  void _ZdlPvRKSt9nothrow_t(void* p, mi_nothrow_t tag)      { MI_UNUSED(tag); mi_free(p); }  // operator delete(void*, std::nothrow_t const&)
+  void _ZdaPvRKSt9nothrow_t(void* p, mi_nothrow_t tag)      { MI_UNUSED(tag); mi_free(p); }  // operator delete[](void*, std::nothrow_t const&)
+  void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&)
+  void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&)
+
+  #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__))  // pr #1257
+    void* _Znwm(size_t n)                             MI_FORWARD1(mi_new,n)  // new 64-bit
+    void* _Znam(size_t n)                             MI_FORWARD1(mi_new,n)  // new[] 64-bit
+    void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
+    void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
+    void* _ZnwmSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al)
+    void* _ZnamSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al)
+    void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
+    void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
+  #elif (MI_INTPTR_SIZE==4)
+    void* _Znwj(size_t n)                             MI_FORWARD1(mi_new,n)  // new 64-bit
+    void* _Znaj(size_t n)                             MI_FORWARD1(mi_new,n)  // new[] 64-bit
+    void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
+    void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
+    void* _ZnwjSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al)
+    void* _ZnajSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al)
+    void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
+    void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
+  #else
+    #error "define overloads for new/delete for this platform (just for performance, can be skipped)"
+  #endif
+#endif // __cplusplus
+
+// ------------------------------------------------------
+// Further Posix & Unix functions definitions
+// ------------------------------------------------------
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef MI_OSX_IS_INTERPOSED
+  // Forward Posix/Unix calls as well
+  void*  reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize)
+  size_t malloc_size(const void* p)        MI_FORWARD1(mi_usable_size,p)
+  #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__)
+  size_t malloc_usable_size(void *p)       MI_FORWARD1(mi_usable_size,p)
+  #else
+  size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p)
+  #endif
+
+  // No forwarding here due to aliasing/name mangling issues
+  void*  valloc(size_t size)               { return mi_valloc(size); }
+  void   vfree(void* p)                    { mi_free(p); }
+  size_t malloc_good_size(size_t size)     { return mi_malloc_good_size(size); }
+  int    posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); }
+
+  // `aligned_alloc` is only available when __USE_ISOC11 is defined.
+  // Note: it seems __USE_ISOC11 is not defined in musl (and perhaps other libc's) so we only check
+  // for it if using glibc.
+  // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot
+  // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9.
+  // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it
+  // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we  can avoid overriding it ourselves.
+  #if !defined(__GLIBC__) || __USE_ISOC11
+  void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }
+  #endif
+#endif
+
+// no forwarding here due to aliasing/name mangling issues
+void  cfree(void* p)                                    { mi_free(p); }
+void* pvalloc(size_t size)                              { return mi_pvalloc(size); }
+void* memalign(size_t alignment, size_t size)           { return mi_memalign(alignment, size); }
+#if !defined(_WIN32)
+void* _aligned_malloc(size_t size, size_t alignment)    { return mi_malloc_aligned(size,alignment); }
+#endif
+void* reallocarray(void* p, size_t count, size_t size)  { return mi_reallocarray(p, count, size); }
+// some systems define reallocarr so mark it as a weak symbol (#751)
+mi_decl_weak int reallocarr(void* p, size_t count, size_t size)    { return mi_reallocarr(p, count, size); }
+
+#if defined(__wasi__)
+  // forward __libc interface (see PR #667)
+  void* __libc_malloc(size_t size)                      MI_FORWARD1(mi_malloc, size)
+  void* __libc_calloc(size_t count, size_t size)        MI_FORWARD2(mi_calloc, count, size)
+  void* __libc_realloc(void* p, size_t size)            MI_FORWARD2(mi_realloc, p, size)
+  void  __libc_free(void* p)                            MI_FORWARD0(mi_free, p)
+  void* __libc_memalign(size_t alignment, size_t size)  { return mi_memalign(alignment, size); }
+
+#elif defined(__linux__)
+  // forward __libc interface (needed for glibc-based and musl-based Linux distributions)
+  void* __libc_malloc(size_t size)                      MI_FORWARD1(mi_malloc,size)
+  void* __libc_calloc(size_t count, size_t size)        MI_FORWARD2(mi_calloc,count,size)
+  void* __libc_realloc(void* p, size_t size)            MI_FORWARD2(mi_realloc,p,size)
+  void  __libc_free(void* p)                            MI_FORWARD0(mi_free,p)
+  void  __libc_cfree(void* p)                           MI_FORWARD0(mi_free,p)
+
+  void* __libc_valloc(size_t size)                      { return mi_valloc(size); }
+  void* __libc_pvalloc(size_t size)                     { return mi_pvalloc(size); }
+  void* __libc_memalign(size_t alignment, size_t size)  { return mi_memalign(alignment,size); }
+  int   __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); }
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
+#pragma GCC visibility pop
+#endif
+
+#endif // MI_MALLOC_OVERRIDE
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-posix.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-posix.c
new file mode 100644
index 00000000..60639ff4
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-posix.c
@@ -0,0 +1,202 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// ------------------------------------------------------------------------
+// mi prefixed publi definitions of various Posix, Unix, and C++ functions
+// for convenience and used when overriding these functions.
+// ------------------------------------------------------------------------
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+
+// ------------------------------------------------------
+// Posix & Unix functions definitions
+// ------------------------------------------------------
+
+#include <errno.h>
+#include <string.h>  // memset
+#include <stdlib.h>  // getenv
+
+#ifdef _MSC_VER
+#pragma warning(disable:4996)  // getenv _wgetenv
+#endif
+
+#ifndef EINVAL
+#define EINVAL 22
+#endif
+#ifndef ENOMEM
+#define ENOMEM 12
+#endif
+
+
+mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {
+  // if (!mi_is_in_heap_region(p)) return 0;
+  return mi_usable_size(p);
+}
+
+mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
+  // if (!mi_is_in_heap_region(p)) return 0;
+  return mi_usable_size(p);
+}
+
+mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {
+  return mi_good_size(size);
+}
+
+void mi_cfree(void* p) mi_attr_noexcept {
+  if (mi_is_in_heap_region(p)) {
+    mi_free(p);
+  }
+}
+
+int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {
+  // Note: The spec dictates we should not modify `*p` on an error. (issue#27)
+  // <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
+  if (p == NULL) return EINVAL;
+  if ((alignment % sizeof(void*)) != 0) return EINVAL;                 // natural alignment
+  // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned`
+  if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL;  // not a power of 2
+  void* q = mi_malloc_aligned(size, alignment);
+  if (q==NULL && size != 0) return ENOMEM;
+  mi_assert_internal(_mi_is_aligned(q,alignment));
+  *p = q;
+  return 0;
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
+  void* p = mi_malloc_aligned(size, alignment);
+  mi_assert_internal(_mi_is_aligned(p,alignment));
+  return p;
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept {
+  return mi_memalign( _mi_os_page_size(), size );
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {
+  size_t psize = _mi_os_page_size();
+  if (size >= SIZE_MAX - psize) return NULL; // overflow
+  size_t asize = _mi_align_up(size, psize);
+  return mi_malloc_aligned(asize, psize);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
+  // C11 requires the size to be an integral multiple of the alignment, see <https://en.cppreference.com/w/c/memory/aligned_alloc>.
+  // unfortunately, it turns out quite some programs pass a size that is not an integral multiple so skip this check..
+  /* if mi_unlikely((size & (alignment - 1)) != 0) { // C11 requires alignment>0 && integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
+      #if MI_DEBUG > 0
+      _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment);
+      #endif
+      return NULL;
+    }
+  */
+  // C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned
+  void* p = mi_malloc_aligned(size, alignment);
+  mi_assert_internal(_mi_is_aligned(p,alignment));
+  return p;
+}
+
+mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept {  // BSD <https://man.freebsd.org/cgi/man.cgi?query=reallocarray>
+  size_t total;
+  if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
+    errno = EOVERFLOW;
+    return NULL;
+  }
+  void* newp = mi_realloc(p,total);
+  if (newp==NULL) { errno = ENOMEM; }
+  return newp;
+}
+
+mi_decl_nodiscard int mi_reallocarr( void* ptrp, size_t count, size_t size ) mi_attr_noexcept { // NetBSD <https://man.netbsd.org/reallocarr.3>
+  mi_assert(size != 0);
+  mi_assert(ptrp != NULL);
+  if (ptrp == NULL || size == 0) {
+    return (errno = EINVAL);
+  }
+  size_t total;
+  if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
+    return (errno = EOVERFLOW);
+  }
+  void** op = (void**)ptrp;
+  if (total == 0) {
+    free(*op);
+    *op = NULL;
+    return 0;
+  }
+  else {
+    void* newp = mi_realloc(*op,total);
+    if (newp == NULL) { return (errno = ENOMEM); }
+    *op = newp;
+    return 0;
+  }
+}
+
+void* mi__expand(void* p, size_t newsize) mi_attr_noexcept {  // Microsoft
+  void* res = mi_expand(p, newsize);
+  if (res == NULL) { errno = ENOMEM; }
+  return res;
+}
+
+mi_decl_nodiscard mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept {
+  if (s==NULL) return NULL;
+  size_t wlen;
+  for(wlen = 0; s[wlen] != 0 && wlen < PTRDIFF_MAX; wlen++) { }  // prevent overflow on wlen+1
+  size_t size; 
+  if (mi_mul_overflow(wlen+1, sizeof(wchar_t), &size) || size > PTRDIFF_MAX) return NULL;
+  wchar_t* p = (wchar_t*)mi_malloc(size);
+  if (p != NULL) {
+    _mi_memcpy(p,s,size);
+  }
+  return p;
+}
+
+mi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s)  mi_attr_noexcept {
+  return (unsigned char*)mi_strdup((const char*)s);
+}
+
+int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept {
+  if (size != NULL) *size = 0;
+  if (buf==NULL || name==NULL) return EINVAL;
+  char* p = getenv(name);
+  if (p==NULL) {
+    *buf = NULL;
+  }
+  else {
+    *buf = mi_strdup(p);
+    if (*buf==NULL) return ENOMEM;
+    if (size != NULL) { *size = _mi_strlen(p) + 1; }    // cannot overflow as mi_strdup is limited to PTRDIFF_MAX
+  }
+  return 0;
+}
+
+int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept {
+  if (size != NULL) *size = 0;
+  if (buf==NULL || name==NULL) return EINVAL;  
+#if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP))
+  // not supported
+  *buf = NULL;
+  return EINVAL;
+#else
+  wchar_t* p = (wchar_t*)_wgetenv(name);
+  if (p==NULL) {
+    *buf = NULL;
+  }
+  else {
+    *buf = mi_wcsdup(p);
+    if (*buf==NULL) return ENOMEM;
+    if (size != NULL) { *size = wcslen(p) + 1; }  // cannot overflow as wcsdup is limited to PTRDIFF_MAX
+  }
+  return 0;
+#endif
+}
+
+mi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
+  return mi_recalloc_aligned_at(p, newcount, size, alignment, offset);
+}
+
+mi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
+  return mi_recalloc_aligned(p, newcount, size, alignment);
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc.c
new file mode 100644
index 00000000..aa556bd0
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc.c
@@ -0,0 +1,885 @@
+
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#ifndef _DEFAULT_SOURCE
+#define _DEFAULT_SOURCE   // for realpath() on Linux
+#endif
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/atomic.h"
+#include "mimalloc/prim.h"   // _mi_prim_thread_id()
+
+#include <string.h>      // memset, strlen (for mi_strdup)
+#include <stdlib.h>      // malloc, abort
+
+#define MI_IN_ALLOC_C
+#include "alloc-override.c"
+#include "free.c"
+#undef MI_IN_ALLOC_C
+
+// ------------------------------------------------------
+// Allocation
+// ------------------------------------------------------
+
+// Fast allocation in a page: just pop from the free list.
+// Fall back to generic allocation only if the list is empty.
+// Note: in release mode the (inlined) routine is about 7 instructions with a single test.
+static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept
+{
+  if (page->block_size != 0) { // not the empty theap
+    mi_assert_internal(mi_page_block_size(page) >= size);
+    mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+    mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+  }
+
+  // check the free list
+  mi_block_t* const block = page->free;
+  if mi_unlikely(block == NULL) {
+    return _mi_malloc_generic(theap, size, (zero ? 1 : 0), usable);
+  }
+  mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
+  if (usable != NULL) { *usable = mi_page_usable_block_size(page); };
+
+  // pop from the free list
+  page->free = mi_block_next(page, block);
+  page->used++;
+  mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
+  mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE));
+
+  #if MI_DEBUG>3
+  if (page->free_is_zero && size > sizeof(*block)) {
+    mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block)));
+  }
+  #endif
+
+  // allow use of the block internally
+  // note: when tracking we need to avoid ever touching the MI_PADDING since
+  // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`)
+  const size_t bsize = mi_page_usable_block_size(page);
+  mi_track_mem_undefined(block, bsize);
+
+  #if (MI_STAT>0)
+  if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
+    mi_theap_stat_increase(theap, malloc_normal, bsize);
+    #if (MI_STAT>1)
+    mi_theap_stat_counter_increase(theap, malloc_normal_count, 1);
+    const size_t bin = _mi_bin(bsize);
+    mi_theap_stat_increase(theap, malloc_bins[bin], 1);
+    mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE);
+    #endif
+  }
+  #endif
+
+  // zero the block? note: we need to zero the full block size (issue #63)
+  if mi_likely(!zero) {
+    // #if MI_SECURE
+    block->next = 0;  // don't leak internal data
+    // #endif
+    #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
+      if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); }
+    #endif    
+  }
+  else {
+    if (!page->free_is_zero) {
+      _mi_memzero_aligned(block, bsize);
+    }
+    else {
+      block->next = 0;
+      mi_track_mem_defined(block, bsize);
+    }    
+  }
+
+  #if MI_PADDING // && !MI_TRACK_ENABLED
+    mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize);
+    ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));
+    #if (MI_DEBUG>=2)
+    mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta));
+    #endif
+    mi_track_mem_defined(padding,sizeof(mi_padding_t));  // note: re-enable since mi_page_usable_block_size may set noaccess
+    padding->canary = mi_ptr_encode_canary(page,block,page->keys);
+    padding->delta  = (uint32_t)(delta);
+    #if MI_PADDING_CHECK
+    if (!mi_page_is_huge(page)) {
+      uint8_t* fill = (uint8_t*)padding - delta;
+      const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes
+      for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }
+    }
+    #endif
+  #endif
+
+  return block;
+}
+
+// extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`.
+extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept {
+  return mi_page_malloc_zero(theap, page, size, zero, NULL);
+}
+
+#if MI_GUARDED
+mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;
+#endif
+
+// main allocation primitives for small and generic allocation
+
+// internal small size allocation
+static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept
+{
+  mi_assert(theap != NULL);
+  mi_assert(size <= MI_SMALL_SIZE_MAX);
+  #if MI_DEBUG
+  const uintptr_t tid = _mi_thread_id();
+  mi_assert(theap->tld->thread_id == 0 || theap->tld->thread_id == tid); // theaps are thread local
+  #endif
+  #if (MI_PADDING || MI_GUARDED)
+  if mi_unlikely(size == 0) { size = sizeof(void*); }
+  #endif
+  #if MI_GUARDED
+  if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) {
+    return _mi_theap_malloc_guarded(theap, size, zero);
+  }
+  #endif
+
+  // get page in constant time, and allocate from it
+  mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE);
+  void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, usable);
+  mi_track_malloc(p,size,zero);
+
+  #if MI_DEBUG>3
+  if (p != NULL && zero) {
+    mi_assert_expensive(mi_mem_is_zero(p, size));
+  }
+  #endif
+  return p;
+}
+
+// internal generic allocation
+static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept
+{
+  #if MI_GUARDED
+  #if MI_THEAP_INITASNULL
+  if (theap!=NULL)
+  #endif
+  if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) {
+    return _mi_theap_malloc_guarded(theap, size, zero);
+  }
+  #endif
+  #if !MI_THEAP_INITASNULL
+  mi_assert(theap!=NULL);
+  #endif
+  mi_assert(theap==NULL || theap->tld->thread_id == 0 || theap->tld->thread_id == _mi_thread_id());   // theaps are thread local
+  mi_assert((huge_alignment & 1)==0);
+  void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, usable);  // note: size can overflow but it is detected in malloc_generic
+  mi_track_malloc(p, size, zero);
+
+  #if MI_DEBUG>3
+  if (p != NULL && zero) {
+    mi_assert_expensive(mi_mem_is_zero(p, size));
+  }
+  #endif
+  return p;
+}
+
+// internal small allocation
+static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {
+  #if MI_THEAP_INITASNULL
+  if (theap!=NULL) {
+    return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
+  }
+  else {
+    return mi_theap_malloc_generic(theap, size, zero, 0, usable); // tailcall
+  }
+  #else
+  return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
+  #endif
+}
+
+
+// allocate a small block
+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+  return mi_theap_malloc_small_zero(theap, size, false, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept {
+  return mi_theap_malloc_small(_mi_theap_default(), size);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+  return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL);
+}
+
+// The main internal allocation functions
+static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {
+  // fast path for small objects
+  if mi_likely(size <= MI_SMALL_SIZE_MAX) {
+    mi_assert_internal(huge_alignment == 0);
+    return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
+  }
+  else {
+    return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);
+  }
+}
+
+extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {
+  // fast path for small objects
+  #if MI_THEAP_INITASNULL
+  if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX)
+  #else
+  if mi_likely(size <= MI_SMALL_SIZE_MAX)
+  #endif
+  {
+    mi_assert_internal(huge_alignment == 0);
+    return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
+  }
+  else {
+    return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);
+  }
+}
+
+void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {
+  return _mi_theap_malloc_zero_ex(theap, size, zero, 0, usable);
+}
+
+
+// Main allocation functions
+
+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+  return _mi_theap_malloc_zero(theap, size, false, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {
+   return mi_theap_malloc(_mi_theap_default(), size);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+  return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL);
+}
+
+
+// zero initialized small block
+mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {
+  return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL);
+}
+
+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+  return mi_theap_malloc_small_zero(theap, size, true, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+  return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL);
+}
+
+
+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+  return _mi_theap_malloc_zero(theap, size, true, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {
+  return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+  return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL);
+}
+
+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(count,size,&total)) return NULL;
+  return mi_theap_zalloc(theap,total);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {
+  return mi_theap_calloc(_mi_theap_default(),count,size);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(count, size, &total)) return NULL;
+  return mi_heap_zalloc(heap, total);
+}
+
+// Return usable size
+mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept {
+  return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, usable);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_umalloc(mi_theap_t* theap, size_t size, size_t* usable) mi_attr_noexcept {
+  return _mi_theap_malloc_zero_ex(theap, size, false, 0, usable);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept {
+  return mi_theap_umalloc(_mi_theap_default(), size, usable);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept {
+  return _mi_theap_malloc_zero_ex(_mi_theap_default(), size, true, 0, usable);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(count,size,&total)) return NULL;
+  return mi_uzalloc(total, usable);
+}
+
+// Uninitialized `calloc`
+static mi_decl_restrict void* mi_theap_mallocn(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(count, size, &total)) return NULL;
+  return mi_theap_malloc(theap, total);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {
+  return mi_theap_mallocn(_mi_theap_default(),count,size);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(count, size, &total)) return NULL;
+  return mi_heap_malloc(heap, total);
+}
+
+
+// Expand (or shrink) in place (or fail)
+void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
+  #if MI_PADDING
+  // we do not shrink/expand with padding enabled
+  MI_UNUSED(p); MI_UNUSED(newsize);
+  return NULL;
+  #else
+  if (p == NULL) return NULL;
+  const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand");
+  const size_t size = _mi_usable_size(p,page);
+  if (newsize > size) return NULL;
+  return p; // it fits
+  #endif
+}
+
+void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {
+  // if p == NULL then behave as malloc.
+  // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).
+  // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)
+  const mi_page_t* page;
+  size_t size;
+  if (p==NULL) {
+    page = NULL;
+    size = 0;
+    if (usable_pre!=NULL) { *usable_pre = 0; }
+  }
+  else {
+    page = mi_validate_ptr_page(p,"mi_realloc");
+    size = _mi_usable_size(p,page);
+    if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); }
+  }
+  if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0  // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
+                  && mi_page_heap(page)==_mi_theap_heap(theap))             // and within the same heap
+  {
+    mi_assert_internal(p!=NULL);
+    // todo: do not track as the usable size is still the same in the free; adjust potential padding?
+    // mi_track_resize(p,size,newsize)
+    // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }
+    if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); }
+    return p;  // reallocation still fits and not more than 50% waste
+  }
+  void* newp = mi_theap_umalloc(theap,newsize,usable_post);
+  if mi_likely(newp != NULL) {
+    if (zero && newsize > size) {
+      // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
+      const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
+      _mi_memzero((uint8_t*)newp + start, newsize - start);
+    }
+    else if (newsize == 0) {
+      ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725)
+    }
+    if mi_likely(p != NULL) {
+      const size_t copysize = (newsize > size ? size : newsize);
+      mi_track_mem_defined(p,copysize);  // _mi_useable_size may be too large for byte precise memory tracking..
+      _mi_memcpy(newp, p, copysize);
+      mi_free(p); // only free the original pointer if successful  // todo: optimize since page is known?
+    }
+  }
+  return newp;
+}
+
+mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
+  return _mi_theap_realloc_zero(theap, p, newsize, false, NULL, NULL);
+}
+
+static void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(count, size, &total)) return NULL;
+  return mi_theap_realloc(theap, p, total);
+}
+
+
+// Reallocate but free `p` on errors
+static void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
+  void* newp = mi_theap_realloc(theap, p, newsize);
+  if (newp==NULL && p!=NULL) mi_free(p);
+  return newp;
+}
+
+static void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
+  return _mi_theap_realloc_zero(theap, p, newsize, true, NULL, NULL);
+}
+
+static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {
+  size_t total;
+  if (mi_count_size_overflow(count, size, &total)) return NULL;
+  return mi_theap_rezalloc(theap, p, total);
+}
+
+
+mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {
+  return mi_theap_realloc(_mi_theap_default(),p,newsize);
+}
+
+mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {
+  return mi_theap_reallocn(_mi_theap_default(),p,count,size);
+}
+
+mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {
+  return _mi_theap_realloc_zero(_mi_theap_default(),p,newsize, false, usable_pre, usable_post);
+}
+
+// Reallocate but free `p` on errors
+mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {
+  return mi_theap_reallocf(_mi_theap_default(),p,newsize);
+}
+
+mi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept {
+  return mi_theap_rezalloc(_mi_theap_default(), p, newsize);
+}
+
+mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept {
+  return mi_theap_recalloc(_mi_theap_default(), p, count, size);
+}
+
+
+mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
+  return mi_theap_realloc(_mi_heap_theap(heap), p, newsize);
+}
+
+mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
+  return mi_theap_reallocn(_mi_heap_theap(heap), p, count, size);
+}
+
+// Reallocate but free `p` on errors
+mi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
+  return mi_theap_reallocf(_mi_heap_theap(heap), p, newsize);
+}
+
+mi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
+  return mi_theap_rezalloc(_mi_heap_theap(heap), p, newsize);
+}
+
+mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
+  return mi_theap_recalloc(_mi_heap_theap(heap), p, count, size);
+}
+
+
+
+// ------------------------------------------------------
+// strdup, strndup, and realpath
+// ------------------------------------------------------
+
+// `strdup` using mi_malloc
+mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* theap, const char* s) mi_attr_noexcept {
+  if (s == NULL) return NULL;
+  size_t len = _mi_strlen(s);
+  if (len > MI_MAX_ALLOC_SIZE - 1) return NULL;  // prevent overflow on len+1
+  char* t = (char*)mi_theap_malloc(theap,len+1);
+  if (t == NULL) return NULL;
+  _mi_memcpy(t, s, len);
+  t[len] = 0;
+  return t;
+}
+
+mi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept {
+  return mi_theap_strdup(_mi_theap_default(), s);
+}
+
+mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept {
+  return mi_theap_strdup(_mi_heap_theap(heap), s);
+}
+
+// `strndup` using mi_malloc
+mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* theap, const char* s, size_t n) mi_attr_noexcept {
+  if (s == NULL) return NULL;
+  const size_t len = _mi_strnlen(s,n);  // len <= n
+  if (len > MI_MAX_ALLOC_SIZE - 1) return NULL;  // prevent overflow on len+1
+  char* t = (char*)mi_theap_malloc(theap, len+1);
+  if (t == NULL) return NULL;
+  _mi_memcpy(t, s, len);
+  t[len] = 0;
+  return t;
+}
+
+mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept {
+  return mi_theap_strndup(_mi_theap_default(),s,n);
+}
+
+mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept {
+  return mi_theap_strndup(_mi_heap_theap(heap), s, n);
+}
+
+#ifndef __wasi__
+// `realpath` using mi_malloc
+#ifdef _WIN32
+#ifndef PATH_MAX
+#define PATH_MAX MAX_PATH
+#endif
+
+mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {
+  // todo: use GetFullPathNameW to allow longer file names
+  char buf[PATH_MAX];
+  DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL);
+  if (res == 0) {
+    errno = GetLastError(); return NULL;
+  }
+  else if (res > PATH_MAX) {
+    errno = EINVAL; return NULL;
+  }
+  else if (resolved_name != NULL) {
+    return resolved_name;
+  }
+  else {
+    return mi_theap_strndup(theap, buf, PATH_MAX);
+  }
+}
+
+#else
+
+#include <unistd.h>  // pathconf
+
+static size_t mi_path_max(void) {
+  static _Atomic(size_t) path_max = 0;
+  size_t pmax = mi_atomic_load_acquire(&path_max);
+  if (pmax == 0) {
+    long m = 0;
+    #ifdef _PC_PATH_MAX
+    m = pathconf("/",_PC_PATH_MAX);
+    #endif
+    if (m <= 0) pmax = 4096;      // guess
+    else if (m < 256) pmax = 256; // at least 256
+    else if (m > 64*1024) pmax = 64*1024;  // at most 64 KiB
+    else pmax = m;
+    size_t expected = 0;
+    mi_atomic_cas_strong_acq_rel(&path_max, &expected, pmax);
+  }
+  return pmax;
+}
+
+char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {
+  if (resolved_name != NULL) {
+    return realpath(fname,resolved_name);
+  }
+  else {
+  /*
+    char* rname = realpath(fname, NULL);
+    if (rname == NULL) return NULL;
+    char* result = mi_heap_strdup(heap, rname);
+    mi_cfree(rname);  // note: may leak the original pointer if allocated internally with the system allocator 
+    // note: with ASAN realpath is intercepted and mi_cfree may leak the returned pointer :-(
+    return result;  
+  */
+    const size_t n  = mi_path_max();
+    char* const buf = (char*)mi_zalloc(n+1);
+    if (buf == NULL) {
+      errno = ENOMEM;
+      return NULL;
+    }
+    char* rname  = realpath(fname,buf);
+    char* result = mi_theap_strndup(theap,rname,n); // ok if `rname==NULL`
+    mi_free(buf);
+    return result;
+  }
+}
+#endif
+
+mi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept {
+  return mi_theap_realpath(_mi_theap_default(),fname,resolved_name);
+}
+
+mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept {
+  return mi_theap_realpath(_mi_heap_theap(heap), fname, resolved_name);
+}
+#endif
+
+/*-------------------------------------------------------
+C++ new and new_aligned
+The standard requires calling into `get_new_handler` and
+throwing the bad_alloc exception on failure. If we compile
+with a C++ compiler we can implement this precisely. If we
+use a C compiler we cannot throw a `bad_alloc` exception
+but we call `exit` instead (i.e. not returning).
+-------------------------------------------------------*/
+
+#ifdef __cplusplus
+#include <new>
+static bool mi_try_new_handler(bool nothrow) {
+  #if defined(_MSC_VER) || (__cplusplus >= 201103L)
+    std::new_handler h = std::get_new_handler();
+  #else
+    std::new_handler h = std::set_new_handler();
+    std::set_new_handler(h);
+  #endif
+  if (h==NULL) {
+    _mi_error_message(ENOMEM, "out of memory in 'new'");
+    #if defined(_CPPUNWIND) || defined(__cpp_exceptions)  // exceptions are not always enabled
+    if (!nothrow) {
+      throw std::bad_alloc();
+    }
+    #else
+    MI_UNUSED(nothrow);
+    #endif
+    return false;
+  }
+  else {
+    h();
+    return true;
+  }
+}
+#else
+typedef void (*std_new_handler_t)(void);
+
+#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER)))  // exclude clang-cl, see issue #631
+std_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) {
+  return NULL;
+}
+static std_new_handler_t mi_get_new_handler(void) {
+  return _ZSt15get_new_handlerv();
+}
+#else
+// note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`.
+static std_new_handler_t mi_get_new_handler(void) {
+  return NULL;
+}
+#endif
+
+static bool mi_try_new_handler(bool nothrow) {
+  std_new_handler_t h = mi_get_new_handler();
+  if (h==NULL) {
+    _mi_error_message(ENOMEM, "out of memory in 'new'");
+    if (!nothrow) {
+      abort();  // cannot throw in plain C, use abort
+    }
+    return false;
+  }
+  else {
+    h();
+    return true;
+  }
+}
+#endif
+
+static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) {
+  void* p = NULL;
+  while(p == NULL && mi_try_new_handler(nothrow)) {
+    p = mi_theap_malloc(theap,size);
+  }
+  return p;
+}
+
+static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {
+  return mi_theap_try_new(_mi_theap_default(), size, nothrow);
+}
+
+static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) {
+  return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow);
+}
+
+
+mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {
+  void* p = mi_theap_malloc(theap,size);
+  if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false);
+  return p;
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) {
+  return mi_theap_alloc_new(_mi_theap_default(), size);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) {
+  void* p = mi_heap_malloc(heap, size);
+  if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false);
+  return p;
+}
+
+
+mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {
+  size_t total;
+  if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
+    mi_try_new_handler(false);  // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
+    return NULL;
+  }
+  else {
+    return mi_theap_alloc_new(theap,total);
+  }
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) {
+  return mi_theap_alloc_new_n(_mi_theap_default(), count, size);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) {
+  return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size);
+}
+
+
+mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {
+  void* p = mi_malloc(size);
+  if mi_unlikely(p == NULL) return mi_try_new(size, true);
+  return p;
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) {
+  void* p;
+  do {
+    p = mi_malloc_aligned(size, alignment);
+  }
+  while(p == NULL && mi_try_new_handler(false));
+  return p;
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept {
+  void* p;
+  do {
+    p = mi_malloc_aligned(size, alignment);
+  }
+  while(p == NULL && mi_try_new_handler(true));
+  return p;
+}
+
+mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) {
+  void* q;
+  do {
+    q = mi_realloc(p, newsize);
+  } while (q == NULL && mi_try_new_handler(false));
+  return q;
+}
+
+mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {
+  size_t total;
+  if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) {
+    mi_try_new_handler(false);  // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
+    return NULL;
+  }
+  else {
+    return mi_new_realloc(p, total);
+  }
+}
+
+#if MI_GUARDED
+// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true).
+// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)
+// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`
+
+static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) {
+  // TODO: we can still make padding work by moving it out of the guard page area
+  mi_page_t* const page = _mi_ptr_page(block);
+  mi_page_set_has_interior_pointers(page, true);
+  block->next = MI_BLOCK_TAG_GUARDED;
+
+  // set guard page at the end of the block
+  const size_t block_size = mi_page_block_size(page);  // must use `block_size` to match `mi_free_local`
+  const size_t os_page_size = _mi_os_page_size();
+  mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));
+  if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {
+    // should never happen
+    mi_free(block);
+    return NULL;
+  }
+  uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;
+  // note: the alignment of the guard page relies on blocks being os_page_size aligned which
+  // is ensured in `mi_arena_page_alloc_fresh`.  
+  mi_assert_internal(_mi_is_aligned(block, os_page_size));
+  mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));
+  if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) {
+    const bool ok = _mi_os_protect(guard_page, os_page_size);
+    if mi_unlikely(!ok) {
+      _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size);
+    }
+  }
+  else {
+    _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size);
+  }
+
+  // align pointer just in front of the guard page
+  size_t offset = block_size - os_page_size - obj_size;
+  mi_assert_internal(offset > sizeof(mi_block_t));
+  if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) {
+    // give up to place it right in front of the guard page if the offset is too large for unalignment
+    offset = MI_PAGE_MAX_OVERALLOC_ALIGN;
+  }
+  uint8_t* const p = (uint8_t*)block + offset;
+  mi_assert_internal(p == guard_page - obj_size);
+  mi_track_align(block, p, offset, obj_size);
+  mi_track_mem_defined(block, sizeof(mi_block_t));
+  return p;
+}
+
+mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept
+{
+  // allocate multiple of page size ending in a guard page
+  // ensure minimal alignment requirement?
+  if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) {  // check up front so the `req_size` won't overflow    
+    _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size);
+    return NULL;
+  }
+  const size_t os_page_size = _mi_os_page_size();
+  const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));
+  const size_t bsize    = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);
+  const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);  
+  mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, NULL);
+  if (block==NULL) return NULL;
+  void* const p = mi_block_ptr_set_guarded(block, obj_size);
+  if (p == NULL) return p;
+  if (zero) { 
+    _mi_memzero_aligned(p,obj_size);  // we have to zero here as padding might have written here (if the blocksize > reqsize + os_page_size)
+  }
+
+  // stats
+  mi_track_malloc(p, obj_size, zero);    
+  if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); }
+  mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1);
+  #if MI_STAT>1
+  // adjust stats to only count the allocated size of the block (and not the guard page)
+  mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size);
+  mi_theap_stat_increase(theap, malloc_requested, size);
+  #endif
+  #if MI_DEBUG>3
+  if (zero) {
+    mi_assert_expensive(mi_mem_is_zero(p, size));
+  }
+  #endif
+  return p;
+}
+#endif
+
+// ------------------------------------------------------
+// ensure explicit external inline definitions are emitted!
+// ------------------------------------------------------
+
+#ifdef __cplusplus
+void* _mi_externs[] = {
+  (void*)&_mi_page_malloc_zero,
+  (void*)&_mi_theap_malloc_zero,
+  (void*)&_mi_theap_malloc_zero_ex,
+  (void*)&mi_theap_malloc,
+  (void*)&mi_theap_zalloc,
+  (void*)&mi_theap_malloc_small,
+  (void*)&mi_malloc,
+  (void*)&mi_malloc_small,
+  (void*)&mi_zalloc,
+  (void*)&mi_zalloc_small,
+  (void*)&mi_heap_malloc,
+  (void*)&mi_heap_malloc_small,
+  (void*)&mi_malloc_aligned
+  // (void*)&mi_theap_alloc_new,
+  // (void*)&mi_theap_alloc_new_n
+};
+#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena-meta.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena-meta.c
new file mode 100644
index 00000000..cee38caa
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena-meta.c
@@ -0,0 +1,179 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2019-2024, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+/* ----------------------------------------------------------------------------
+  We have a special "mini" allocator just for allocation of meta-data like
+  the theap (`mi_theap_t`) or thread-local data (`mi_tld_t`).
+
+  We reuse the bitmap of the arena's for allocation of 64b blocks inside
+  an arena slice (64KiB).
+  We always ensure that meta data is zero'd (we zero on `free`)
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "bitmap.h"
+
+/* -----------------------------------------------------------
+  Meta data allocation
+----------------------------------------------------------- */
+
+#define MI_META_PAGE_SIZE         MI_ARENA_SLICE_SIZE
+#define MI_META_PAGE_ALIGN        MI_ARENA_SLICE_ALIGN
+
+// large enough such that META_MAX_SIZE > 4k (even on 32-bit)
+#define MI_META_BLOCK_SIZE        (1 << (16 - MI_BCHUNK_BITS_SHIFT))        // 128 on 64-bit
+#define MI_META_BLOCK_ALIGN       MI_META_BLOCK_SIZE
+#define MI_META_BLOCKS_PER_PAGE   (MI_META_PAGE_SIZE / MI_META_BLOCK_SIZE)  // 512
+#define MI_META_MAX_SIZE          (MI_BCHUNK_SIZE * MI_META_BLOCK_SIZE)
+
+#if MI_META_MAX_SIZE <= 4096
+#error "max meta object size should be at least 4KiB"
+#endif
+
+typedef struct mi_meta_page_s  {
+  _Atomic(struct mi_meta_page_s*)  next;    // a linked list of meta-data pages (never released)
+  mi_memid_t                       memid;   // provenance of the meta-page memory itself
+  mi_bbitmap_t                     blocks_free;  // a small bitmap with 1 bit per block.
+} mi_meta_page_t;
+
+static mi_decl_cache_align _Atomic(mi_meta_page_t*)  mi_meta_pages = MI_ATOMIC_VAR_INIT(NULL);
+
+
+#if MI_DEBUG > 1
+static mi_meta_page_t* mi_meta_page_of_ptr(void* p, size_t* block_idx) {
+  mi_meta_page_t* mpage = (mi_meta_page_t*)((uint8_t*)_mi_align_down_ptr(p,MI_META_PAGE_ALIGN) + _mi_os_secure_guard_page_size());
+  if (block_idx != NULL) {
+    *block_idx = ((uint8_t*)p - (uint8_t*)mpage) / MI_META_BLOCK_SIZE;
+  }
+  return mpage;
+}
+#endif
+
+static mi_meta_page_t* mi_meta_page_next( mi_meta_page_t* mpage ) {
+  return mi_atomic_load_ptr_acquire(mi_meta_page_t, &mpage->next);
+}
+
+static void* mi_meta_block_start( mi_meta_page_t* mpage, size_t block_idx ) {
+  mi_assert_internal(_mi_is_aligned((uint8_t*)mpage - _mi_os_secure_guard_page_size(), MI_META_PAGE_ALIGN));
+  mi_assert_internal(block_idx < MI_META_BLOCKS_PER_PAGE);
+  void* p = ((uint8_t*)mpage - _mi_os_secure_guard_page_size() + (block_idx * MI_META_BLOCK_SIZE));
+  mi_assert_internal(mpage == mi_meta_page_of_ptr(p,NULL));
+  return p;
+}
+
+// allocate a fresh meta page and add it to the global list.
+static mi_meta_page_t* mi_meta_page_zalloc(void) {
+  // allocate a fresh arena slice
+  // note: careful with _mi_subproc as it may recurse into mi_tld and meta_page_zalloc again.. (same with _mi_os_numa_node()...)
+  mi_memid_t memid;
+  uint8_t* base = (uint8_t*)_mi_arenas_alloc_aligned(mi_heap_main(), MI_META_PAGE_SIZE, MI_META_PAGE_ALIGN, 0,
+                                                                    true /* commit*/, (MI_SECURE==0) /* allow large? */,
+                                                                    NULL /* req arena */, 0 /* thread_seq */, -1 /* numa node */, &memid);
+  if (base == NULL) return NULL;
+  mi_assert_internal(_mi_is_aligned(base,MI_META_PAGE_ALIGN));
+  if (!memid.initially_zero) {
+    _mi_memzero_aligned(base, MI_ARENA_SLICE_SIZE);
+  }
+
+  // guard pages
+  #if MI_SECURE >= 1
+  _mi_os_secure_guard_page_set_at(base, memid);
+  _mi_os_secure_guard_page_set_before(base + MI_META_PAGE_SIZE, memid);
+  #endif
+
+  // initialize the page and free block bitmap
+  mi_meta_page_t* mpage = (mi_meta_page_t*)(base + _mi_os_secure_guard_page_size());
+  mpage->memid = memid;
+  mi_bbitmap_init(&mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */);
+  const size_t mpage_size  = offsetof(mi_meta_page_t,blocks_free) + mi_bbitmap_size(MI_META_BLOCKS_PER_PAGE, NULL);
+  const size_t info_blocks = _mi_divide_up(mpage_size,MI_META_BLOCK_SIZE);
+  const size_t guard_blocks = _mi_divide_up(_mi_os_secure_guard_page_size(), MI_META_BLOCK_SIZE);
+  mi_assert_internal(info_blocks + 2*guard_blocks < MI_META_BLOCKS_PER_PAGE);
+  mi_bbitmap_unsafe_setN(&mpage->blocks_free, info_blocks + guard_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks - 2*guard_blocks);
+
+  // push atomically in front of the meta page list
+  // (note: there is no ABA issue since we never free meta-pages)
+  mi_meta_page_t* old = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);
+  do {
+    mi_atomic_store_ptr_release(mi_meta_page_t, &mpage->next, old);
+  } while(!mi_atomic_cas_ptr_weak_acq_rel(mi_meta_page_t,&mi_meta_pages,&old,mpage));
+  return mpage;
+}
+
+
+// allocate meta-data
+mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid )
+{
+  mi_assert_internal(pmemid != NULL);
+  size = _mi_align_up(size,MI_META_BLOCK_SIZE);
+  if (size == 0 || size > MI_META_MAX_SIZE) return NULL;
+  const size_t block_count = _mi_divide_up(size,MI_META_BLOCK_SIZE);
+  mi_assert_internal(block_count > 0 && block_count < MI_BCHUNK_BITS);
+  mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);
+  mi_meta_page_t* mpage = mpage0;
+  while (mpage != NULL) {
+    size_t block_idx;
+    if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
+      // found and claimed `block_count` blocks
+      *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
+      return mi_meta_block_start(mpage,block_idx);
+    }
+    else {
+      mpage = mi_meta_page_next(mpage);
+    }
+  }
+  // failed to find space in existing pages
+  if (mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages) != mpage0) {
+    // the page list was updated by another thread in the meantime, retry
+    return _mi_meta_zalloc(size,pmemid);
+  }
+  // otherwise, allocate a fresh metapage and try once more
+  mpage = mi_meta_page_zalloc();
+  if (mpage != NULL) {
+    size_t block_idx;
+    if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
+      // found and claimed `block_count` blocks
+      *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
+      return mi_meta_block_start(mpage,block_idx);
+    }
+  }
+  // if all this failed, allocate from the OS
+  return _mi_os_alloc(size, pmemid);
+}
+
+// free meta-data
+mi_decl_noinline void _mi_meta_free(void* p, size_t size, mi_memid_t memid) {
+  if (p==NULL) return;
+  if (memid.memkind == MI_MEM_META) {
+    mi_assert_internal(_mi_divide_up(size, MI_META_BLOCK_SIZE) == memid.mem.meta.block_count);
+    const size_t block_count = memid.mem.meta.block_count;
+    const size_t block_idx   = memid.mem.meta.block_index;
+    mi_meta_page_t* mpage = (mi_meta_page_t*)memid.mem.meta.meta_page;
+    mi_assert_internal(mi_meta_page_of_ptr(p,NULL) == mpage);
+    mi_assert_internal(block_idx + block_count <= MI_META_BLOCKS_PER_PAGE);
+    mi_assert_internal(mi_bbitmap_is_clearN(&mpage->blocks_free, block_idx, block_count));
+    // we zero on free (and on the initial page allocation) so we don't need a "dirty" map
+    _mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE);
+    mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count);
+  }
+  else {
+    _mi_arenas_free(p,size,memid);
+  }
+}
+
+// used for debug output
+bool _mi_meta_is_meta_page(void* p)
+{
+  mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t, &mi_meta_pages);
+  mi_meta_page_t* mpage = mpage0;
+  while (mpage != NULL) {
+    if ((void*)mpage == p) return true;
+    mpage = mi_meta_page_next(mpage);
+  }
+  return false;
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena.c
new file mode 100644
index 00000000..a5686d55
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena.c
@@ -0,0 +1,2514 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2019-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+/* ----------------------------------------------------------------------------
+"Arenas" are fixed area's of OS memory from which we can allocate
+large blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 64KiB).
+In contrast to the rest of mimalloc, the arenas are shared between
+threads and need to be accessed using atomic operations.
+
+Arenas are also used to for huge OS page (1GiB) reservations or for reserving
+OS memory upfront which can be improve performance or is sometimes needed
+on embedded devices. We can also employ this with WASI or `sbrk` systems
+to reserve large arenas upfront and be able to reuse the memory more effectively.
+
+The arena allocation needs to be thread safe and we use an atomic bitmap to allocate.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"
+#include "bitmap.h"
+
+/* -----------------------------------------------------------
+  Arena id's
+----------------------------------------------------------- */
+
+mi_arena_id_t _mi_arena_id_none(void) {
+  return NULL;
+}
+
+mi_arena_t* _mi_arena_from_id(mi_arena_id_t id) {
+  mi_arena_t* const arena = (mi_arena_t*)id;
+  mi_assert_internal(arena==NULL || arena->parent==NULL); // id's should never point to sub-arena's
+  return arena;
+}
+
+mi_arena_id_t mi_arena_id_from_arena(mi_arena_t* arena) {
+  mi_assert_internal(arena==NULL || arena->parent==NULL);
+  return (arena==NULL ? _mi_arena_id_none() : (mi_arena_id_t)arena);
+}
+
+
+static bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_t* req_arena) {
+  if (arena == req_arena) return true;                         // they match
+  if (arena == NULL) return false;
+  if (req_arena == NULL && !arena->is_exclusive) return true;  // or the arena is not exclusive, and we didn't request a specific one
+  if (arena->parent != NULL && arena->parent == req_arena) return true;  // sub-arena? (note that req_arena is never a sub arena)
+  return false;
+}
+
+bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena) {
+  if (memid.memkind == MI_MEM_ARENA) {
+    return mi_arena_is_suitable(memid.mem.arena.arena, request_arena);
+  }
+  else {
+    return mi_arena_is_suitable(NULL, request_arena);
+  }
+}
+
+size_t mi_arenas_get_count(mi_subproc_t* subproc) {
+  return mi_atomic_load_relaxed(&subproc->arena_count);
+}
+
+mi_arena_t* mi_arena_from_index(mi_subproc_t* subproc, size_t idx) {
+  mi_assert_internal(idx < mi_arenas_get_count(subproc));
+  return mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[idx]);
+}
+
+static size_t mi_arena_info_slices(mi_arena_t* arena) {
+  return arena->info_slices;
+}
+
+#if MI_DEBUG > 1
+static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page) {
+  mi_assert(arena->arena_idx < MI_MAX_ARENAS);
+  mi_arena_pages_t* arena_pages = heap->arena_pages[arena->arena_idx];
+  return (page->memid.memkind == MI_MEM_ARENA &&
+          page->memid.mem.arena.arena == arena &&
+          arena_pages != NULL &&
+          mi_bitmap_is_setN(arena_pages->pages, page->memid.mem.arena.slice_index, 1));
+}
+#endif
+
+size_t mi_arena_min_alignment(void) {
+  return MI_ARENA_SLICE_ALIGN;
+}
+
+size_t mi_arena_min_size(void) {
+  return MI_ARENA_MIN_SIZE;
+}
+
+static size_t mi_arena_max_object_size(void) {
+  size_t max_size = mi_option_get_size(mi_option_arena_max_object_size);
+  max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE);
+  if (max_size <= MI_ARENA_MIN_OBJ_SIZE) {
+    return MI_ARENA_MIN_OBJ_SIZE;
+  }
+  else if (max_size >= MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE) {  // minus a bchunk to accommodate meta info
+    return (MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE);
+  }
+  else {
+    return max_size;
+  }
+}
+
+mi_decl_nodiscard static bool mi_arena_commit(mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) {
+  if (arena != NULL && arena->commit_fun != NULL) {
+    return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg);
+  }
+  else if (already_committed > 0) {
+    return _mi_os_commit_ex(start, size, is_zero, already_committed);
+  }
+  else {
+    return _mi_os_commit(start, size, is_zero);
+  }
+}
+
+
+
+/* -----------------------------------------------------------
+  Util
+----------------------------------------------------------- */
+
+
+// Size of an arena
+static size_t mi_arena_size(mi_arena_t* arena) {
+  return mi_size_of_slices(arena->slice_count);
+}
+
+// Start of the arena memory area
+static uint8_t* mi_arena_start(mi_arena_t* arena) {
+  return ((uint8_t*)arena);
+}
+
+// Start of a slice
+uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) {
+  mi_assert_internal(slice_index < arena->slice_count);
+  return (mi_arena_start(arena) + mi_size_of_slices(slice_index));
+}
+
+mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) {
+  mi_assert_internal(slice_index < arena->slice_count);
+  if (arena->pages_meta != NULL) {
+    mi_page_t* const page = &arena->pages_meta[slice_index];
+    #if MI_PAGE_META_ALIGNED_FREE_SMALL
+    // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0)
+    if (page->block_size>0) return page;
+    #else
+    return page;
+    #endif
+  }
+  // fall through (for MI_PAGE_META_ALIGNED_FREE_SMALL)
+  return (mi_page_t*)mi_arena_slice_start(arena,slice_index);
+}
+
+// Arena area
+void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) {
+  if (size != NULL) *size = 0;
+  mi_arena_t* arena = _mi_arena_from_id(arena_id);
+  if (arena == NULL) return NULL;
+  if (size != NULL) {
+    mi_assert_internal(mi_size_of_slices(arena->slice_count) <= arena->total_size);
+    *size = arena->total_size;
+  }
+  return mi_arena_start(arena);
+}
+
+
+// Create an arena memid
+static mi_memid_t mi_memid_create_arena(mi_arena_t* arena, size_t slice_index, size_t slice_count) {
+  mi_assert_internal(slice_index < UINT32_MAX);
+  mi_assert_internal(slice_count < UINT32_MAX);
+  mi_assert_internal(slice_count > 0);
+  mi_assert_internal(slice_index < arena->slice_count);
+  mi_memid_t memid = _mi_memid_create(MI_MEM_ARENA);
+  memid.mem.arena.arena = arena;
+  memid.mem.arena.slice_index = (uint32_t)slice_index;
+  memid.mem.arena.slice_count = (uint32_t)slice_count;
+  return memid;
+}
+
+// get the arena and slice span
+static mi_arena_t* mi_arena_from_memid(mi_memid_t memid, size_t* slice_index, size_t* slice_count) {
+  mi_assert_internal(memid.memkind == MI_MEM_ARENA);
+  mi_arena_t* arena = memid.mem.arena.arena;
+  if (slice_index!=NULL) { *slice_index = memid.mem.arena.slice_index; }
+  if (slice_count!=NULL) { *slice_count = memid.mem.arena.slice_count; }
+  return arena;
+}
+
+static size_t mi_page_full_size(mi_page_t* page) {
+  if (page->memid.memkind == MI_MEM_ARENA) {
+    return page->memid.mem.arena.slice_count * MI_ARENA_SLICE_SIZE;
+  }
+  else if (mi_memid_is_os(page->memid) || page->memid.memkind == MI_MEM_EXTERNAL) {
+    mi_assert_internal((uint8_t*)page->memid.mem.os.base <= (uint8_t*)page);
+    const ptrdiff_t presize = (uint8_t*)page - (uint8_t*)page->memid.mem.os.base;
+    mi_assert_internal((ptrdiff_t)page->memid.mem.os.size >= presize);
+    return (presize > (ptrdiff_t)page->memid.mem.os.size ? 0 : page->memid.mem.os.size - presize);
+  }
+  else {
+    return 0;
+  }
+}
+
+
+/* -----------------------------------------------------------
+  Arena Allocation
+----------------------------------------------------------- */
+
+static mi_decl_noinline void* mi_arena_try_alloc_at(
+  mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid)
+{
+  size_t slice_index;
+  if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL;
+
+  // claimed it!
+  void* p = mi_arena_slice_start(arena, slice_index);
+  *memid = mi_memid_create_arena(arena, slice_index, slice_count);
+  memid->is_pinned = arena->memid.is_pinned;
+
+  // set the dirty bits and track which slices become accessible
+  size_t touched_slices = slice_count;
+  if (arena->memid.initially_zero) {
+    size_t already_dirty = 0;
+    memid->initially_zero = mi_bitmap_setN(arena->slices_dirty, slice_index, slice_count, &already_dirty);
+    mi_assert_internal(already_dirty <= touched_slices);
+    touched_slices -= already_dirty;
+  }
+
+  // set commit state
+  if (commit) {
+    // commit requested, but the range may not be committed as a whole: ensure it is committed now
+    const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count);
+    if (already_committed < slice_count) {
+      // not all committed, try to commit now
+      bool commit_zero = false;
+      if (!mi_arena_commit(arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) {
+        // if the commit fails, release ownership, and return NULL;
+        // note: this does not roll back dirty bits but that is ok.
+        mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);
+        return NULL;
+      }
+      if (commit_zero) {
+        memid->initially_zero = true;
+      }
+
+      // set the commit bits
+      mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL);
+
+      // committed
+      #if MI_DEBUG > 1
+      if (memid->initially_zero) {
+        if (!mi_mem_is_zero(p, mi_size_of_slices(slice_count))) {
+          _mi_error_message(EFAULT, "internal error: arena allocation was not zero-initialized!\n");
+          memid->initially_zero = false;
+        }
+      }
+      #endif
+    }
+    else {
+      // already fully committed.
+      _mi_os_reuse(p, mi_size_of_slices(slice_count));
+      // if the OS has overcommit, and this is the first time we access these pages, then
+      // count the commit now (as at arena reserve we didn't count those commits as these are on-demand)
+      if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) {
+        mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices));
+      }
+    }
+
+    mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+    memid->initially_committed = true;
+
+    // tool support
+    if (memid->initially_zero) {
+      mi_track_mem_defined(p, slice_count * MI_ARENA_SLICE_SIZE);
+    }
+    else {
+      mi_track_mem_undefined(p, slice_count * MI_ARENA_SLICE_SIZE);
+    }
+  }
+  else {
+    // no need to commit, but check if it is already fully committed
+    memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);
+    if (!memid->initially_committed) {
+      // partly committed.. adjust stats
+      size_t already_committed_count = 0;
+      mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count);
+      mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
+      mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count));
+    }
+  }
+
+  mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
+  if (commit) { mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); }
+  if (commit) { mi_assert_internal(memid->initially_committed); }
+  mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
+
+  return p;
+}
+
+
+static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);
+
+// try to reserve a fresh arena space
+static bool mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_large, mi_arena_id_t* arena_id)
+{
+  const size_t arena_count = mi_arenas_get_count(subproc);
+  if (arena_count > (MI_MAX_ARENAS - 4)) return false;
+
+  // calc reserve
+  size_t arena_reserve = mi_option_get_size(mi_option_arena_reserve);
+  if (arena_reserve == 0) return false;
+
+  if (!_mi_os_has_virtual_reserve()) {
+    arena_reserve = arena_reserve/4;  // be conservative if virtual reserve is not supported (for WASM for example)
+  }
+  arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_SLICE_SIZE);
+
+  if (arena_count >= 1 && arena_count <= 128) {
+    // scale up the arena sizes exponentially every 8 entries
+    const size_t multiplier = (size_t)1 << _mi_clamp(arena_count/8, 0, 16);
+    size_t reserve = 0;
+    if (!mi_mul_overflow(multiplier, arena_reserve, &reserve)) {
+      arena_reserve = reserve;
+    }
+  }
+
+  // try to accommodate the requested size for huge allocations
+  req_size = _mi_align_up(req_size + MI_ARENA_MAX_CHUNK_OBJ_SIZE, MI_ARENA_MAX_CHUNK_OBJ_SIZE); // over-reserve for meta-info
+  if (arena_reserve < req_size) {
+    arena_reserve = req_size;
+  }
+
+  // check arena bounds
+  const size_t min_reserve = MI_ARENA_MIN_SIZE;
+  const size_t max_reserve = MI_ARENA_MAX_SIZE;   // 16 GiB
+  if (arena_reserve < min_reserve) {
+    arena_reserve = min_reserve;
+  }
+  else if (arena_reserve > max_reserve) {
+    arena_reserve = max_reserve;
+  }
+
+  // should be able to at least handle the current allocation size
+  if (arena_reserve < req_size) return false;
+
+  // commit eagerly?
+  bool arena_commit = false;
+  const bool overcommit = _mi_os_has_overcommit();
+  if (mi_option_get(mi_option_arena_eager_commit) == 2) { arena_commit = overcommit || mi_option_is_enabled(mi_option_allow_large_os_pages); }
+  else if (mi_option_get(mi_option_arena_eager_commit) == 1) { arena_commit = true; }
+
+  // on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice
+  // is actually allocated for the first time it will be counted.
+  const bool adjust = (overcommit && arena_commit);
+  if (adjust) { mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve); }
+  // and try to reserve the arena
+  int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id);
+  if (err != 0) {
+    if (adjust) { mi_subproc_stat_adjust_increase( subproc, committed, arena_reserve); } // roll back
+    // failed to allocate: try a smaller size arena as fallback?
+    const size_t small_arena_reserve = 4 * MI_ARENA_MIN_SIZE; // 128 MiB (or 32 MiB on 32-bit)
+    if (arena_reserve > small_arena_reserve && small_arena_reserve > req_size) {
+      // try again
+      if (adjust) { mi_subproc_stat_adjust_decrease(subproc, committed, small_arena_reserve); }
+      err = mi_reserve_os_memory_ex2(subproc, small_arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id);
+      if (err != 0 && adjust) { mi_subproc_stat_adjust_increase( subproc, committed, small_arena_reserve); } // roll back
+    }
+  }
+  return (err==0);
+}
+
+
+
+
+/* -----------------------------------------------------------
+  Arena iteration
+----------------------------------------------------------- */
+
+static inline bool mi_arena_is_suitable_ex(mi_arena_t* arena, mi_arena_t* req_arena, bool match_numa, int numa_node, bool allow_pinned) {
+  if (!allow_pinned && arena->memid.is_pinned) return false;
+  if (!mi_arena_is_suitable(arena, req_arena)) return false;
+  if (req_arena == NULL) { // if not specific, check numa affinity
+    const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node);
+    if (match_numa) { if (!numa_suitable) return false; }
+               else { if (numa_suitable)  return false; }
+  }
+  return true;
+}
+
+// determine the start of search; important to keep heaps and threads
+// into their own memory regions to reduce contention.
+static size_t mi_arena_start_idx(mi_heap_t* heap, size_t tseq, size_t arena_cycle) {
+  const size_t hseq   = heap->heap_seq;
+  const size_t hcount = mi_atomic_load_relaxed(&heap->subproc->heap_count);
+  if (arena_cycle <= 1)     return 0;
+  if (hseq==0 || hcount<=1 || arena_cycle > 0x8FF) return (tseq % arena_cycle); // common for single heap programs
+
+  // spread heaps evenly among arena's, and then evenly for threads in their fraction
+  size_t start;
+  mi_assert_internal(arena_cycle <= 0x8FF);             // prevent overflow on 32-bit
+  const size_t frac = (arena_cycle * 256) / hcount;     // fraction in the arena_cycle; at most: arena_cycle * 0x100
+  if (frac==0) {
+    // many heaps (> 256 per arena)
+    start = (hseq % arena_cycle);
+  }
+  else {
+    const size_t hspot = (hseq % hcount);  
+    start = (frac * hspot) / 256;           // (arena_cycle * (hseq % hcount)) / hcount
+    if (frac >= 512) {  // at least 2 arena's per heap?
+      start = start + (tseq % (frac/256));
+    }
+  }
+  mi_assert_internal(start < arena_cycle);
+  return start;
+}
+
+#define mi_forall_arenas(heap, req_arena, tseq, name_arena) { \
+  const size_t _arena_count = mi_arenas_get_count(heap->subproc); \
+  const size_t _arena_cycle = (_arena_count == 0 ? 0 : _arena_count - 1); /* first search the arenas below the last one */ \
+  /* always start searching in the arena's below the max */ \
+  const size_t _start = mi_arena_start_idx(heap,tseq,_arena_cycle); \
+  for (size_t _i = 0; _i < _arena_count; _i++) { \
+    mi_arena_t* name_arena; \
+    if (req_arena != NULL) { \
+      name_arena = req_arena; /* if there is a specific req_arena, only search that one */\
+      if (_i > 0) break;      /* only once */ \
+    } \
+    else { \
+      size_t _idx; \
+      if (_i < _arena_cycle) { \
+        _idx = _i + _start; \
+        if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \
+      } \
+      else { \
+        _idx = _i; /* remaining arena's after the cycle */ \
+      } \
+      name_arena = mi_arena_from_index(heap->subproc,_idx); \
+    } \
+    if (name_arena != NULL) \
+    {
+
+#define mi_forall_arenas_end()  \
+    } \
+  } \
+  }
+
+#define mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, numa_node, allow_large, name_arena) \
+  mi_forall_arenas(heap, req_arena,tseq,name_arena) { \
+    if (mi_arena_is_suitable_ex(name_arena, req_arena, match_numa, numa_node, allow_large)) { \
+
+#define mi_forall_suitable_arenas_end() \
+  }} \
+  mi_forall_arenas_end()
+
+/* -----------------------------------------------------------
+  Arena allocation
+----------------------------------------------------------- */
+
+// allocate slices from the arenas
+static mi_decl_noinline void* mi_arenas_try_find_free(
+  mi_heap_t* heap, size_t slice_count, size_t alignment,
+  bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)
+{
+  // mi_assert_internal(slice_count <= mi_slice_count_of_size(MI_ARENA_MAX_CHUNK_OBJ_SIZE));
+  mi_assert(alignment <= MI_ARENA_SLICE_ALIGN);
+  if (alignment > MI_ARENA_SLICE_ALIGN) return NULL;
+
+  // search arena's
+  mi_forall_suitable_arenas(heap, req_arena, tseq, true /* only numa matching */, numa_node, allow_large, arena)
+  {
+    void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid);
+    if (p != NULL) return p;
+  }
+  mi_forall_suitable_arenas_end();
+  if (numa_node < 0) return NULL;
+
+  // search again but now regardless of preferred numa affinity
+  mi_forall_suitable_arenas(heap, req_arena, tseq, false /* numa non-matching now */, numa_node, allow_large, arena)
+  {
+    void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid);
+    if (p != NULL) return p;
+  }
+  mi_forall_suitable_arenas_end();
+  return NULL;
+}
+
+// Allocate slices from the arena's -- potentially allocating a fresh arena
+static mi_decl_noinline void* mi_arenas_try_alloc(
+  mi_heap_t* heap,
+  size_t slice_count, size_t alignment,
+  bool commit, bool allow_large,
+  mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)
+{
+  // mi_assert(slice_count <= MI_ARENA_MAX_CHUNK_OBJ_SLICES);
+  mi_assert(alignment <= MI_ARENA_SLICE_ALIGN);
+  void* p;
+
+  // not too large?
+  if (slice_count * MI_ARENA_SLICE_SIZE > MI_ARENA_MAX_SIZE) return NULL;
+
+  // try to find free slices in the arena's
+  p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid);
+  if (p != NULL) return p;
+
+  // did we need a specific arena?
+  if (req_arena != NULL) return NULL;
+
+  // don't create arena's while preloading (todo: or should we?)
+  if (_mi_preloading()) return NULL;
+
+  // don't create arena's if OS allocation is disallowed
+  if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL;
+
+  // otherwise, try to reserve a new arena -- but one thread at a time.. (todo: allow 2 or 4 to reduce contention?)
+  mi_subproc_t* const subproc = heap->subproc;
+  const size_t arena_count = mi_arenas_get_count(subproc);
+  mi_lock(&subproc->arena_reserve_lock) {
+    if (arena_count == mi_arenas_get_count(subproc)) {
+      // we are the first to enter the lock, reserve a fresh arena
+      mi_arena_id_t arena_id = _mi_arena_id_none();
+      mi_arena_reserve(subproc, mi_size_of_slices(slice_count), allow_large, &arena_id);
+    }
+    else {
+      // another thread already reserved a new arena
+    }
+  }
+  // try once more to allocate in the new arena
+  mi_assert_internal(req_arena == NULL);
+  p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid);
+  if (p != NULL) return p;
+
+  return NULL;
+}
+
+// Allocate from the OS (if allowed)
+static void* mi_arena_os_alloc_aligned(
+  size_t size, size_t alignment, size_t align_offset,
+  bool commit, bool allow_large,
+  mi_arena_id_t req_arena_id, mi_memid_t* memid)
+{
+  // if we cannot use OS allocation, return NULL
+  if (mi_option_is_enabled(mi_option_disallow_os_alloc) || req_arena_id != _mi_arena_id_none()) {
+    errno = ENOMEM;
+    return NULL;
+  }
+
+  if (align_offset > 0) {
+    return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid);
+  }
+  else {
+    return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);
+  }
+}
+
+
+// Allocate large sized memory
+void* _mi_arenas_alloc_aligned( mi_heap_t* heap,
+  size_t size, size_t alignment, size_t align_offset,
+  bool commit, bool allow_large,
+  mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)
+{
+  mi_assert_internal(memid != NULL);
+  mi_assert_internal(size > 0);
+
+  // try to allocate in an arena if the alignment is small enough and the object is not too small (as for theap meta data)
+  if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) &&                // is arena allocation allowed?
+      size >= MI_ARENA_MIN_OBJ_SIZE && size <= mi_arena_max_object_size() &&  // and not too small or too large
+      alignment <= MI_ARENA_SLICE_ALIGN && align_offset == 0)                 // and good alignment
+  {
+    const size_t slice_count = mi_slice_count_of_size(size);
+    void* p = mi_arenas_try_alloc(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid);
+    if (p != NULL) return p;
+  }
+
+  // fall back to the OS
+  void* p = mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena, memid);
+  return p;
+}
+
+void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)
+{
+  return _mi_arenas_alloc_aligned(heap, size, MI_ARENA_SLICE_SIZE, 0, commit, allow_large, req_arena, tseq, numa_node, memid);
+}
+
+
+
+/* -----------------------------------------------------------
+  Arena page allocation
+----------------------------------------------------------- */
+
+// release ownership of a page. This may free the page if all blocks were concurrently
+// freed in the meantime. Returns true if the page was freed.
+static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap) {
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);
+  mi_thread_free_t tf_new;
+  mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);
+  do {
+    mi_assert_internal(mi_tf_is_owned(tf_old));
+    while mi_unlikely(mi_tf_block(tf_old) != NULL) {
+      _mi_page_free_collect(page, false);  // update used
+      if (mi_page_all_free(page)) {        // it may become free just before unowning it
+        _mi_arenas_page_unabandon(page, current_theap);
+        _mi_arenas_page_free(page, current_theap);
+        return true;
+      }
+      tf_old = mi_atomic_load_relaxed(&page->xthread_free);
+    }
+    mi_assert_internal(mi_tf_block(tf_old)==NULL);
+    tf_new = mi_tf_create(NULL, false);
+  } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new));
+  return false;
+}
+
+
+static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena, bool* keep_abandoned) {
+  // found an abandoned page of the right size
+  mi_page_t* const page  = mi_arena_page_at_slice(arena, slice_index);
+  // can we claim ownership?
+  if (!mi_page_claim_ownership(page)) {
+    // there was a concurrent free that reclaims this page ..
+    // we need to keep it in the abandoned map as the free will call `mi_arena_page_unabandon`,
+    // and wait for readers (us!) to finish. This is why it is very important to set the abandoned
+    // bit again (or otherwise the unabandon will never stop waiting).
+    *keep_abandoned = true;
+    return false;
+  }
+  else {
+    // yes, we can reclaim it, keep the abandoned map entry clear
+    *keep_abandoned = false;
+    return true;
+  }
+}
+
+// allocate initial arena_pages from the main heap
+static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena);
+
+static mi_arena_pages_t* mi_heap_arena_pages(mi_heap_t* heap, mi_arena_t* arena) {
+  mi_assert_internal(arena!=NULL);
+  mi_assert_internal(heap!=NULL);
+  mi_assert(arena->arena_idx < MI_MAX_ARENAS);
+  return mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]);
+}
+
+static mi_arena_t* mi_page_arena_pages(mi_page_t* page, size_t* slice_index, size_t* slice_count, mi_arena_pages_t** parena_pages) {
+  // todo: maybe store the arena* directly in the page?
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_arena_t* const arena = mi_arena_from_memid(page->memid, slice_index, slice_count);
+  mi_assert_internal(arena != NULL);
+  if (parena_pages != NULL) {
+    mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(mi_page_heap(page), arena);
+    mi_assert_internal(arena_pages != NULL);
+    mi_assert_internal(slice_index==NULL || mi_bitmap_is_set(arena_pages->pages, *slice_index));
+    *parena_pages = arena_pages;
+  }
+  return arena;
+}
+
+static mi_arena_pages_t* mi_heap_ensure_arena_pages(mi_heap_t* heap, mi_arena_t* arena) {
+  mi_assert_internal(arena!=NULL);
+  mi_assert_internal(heap!=NULL);
+  mi_assert(arena->arena_idx < MI_MAX_ARENAS);
+  mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);
+  if (arena_pages==NULL) {
+    mi_lock(&heap->arena_pages_lock) {
+      arena_pages = mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]);
+      if (arena_pages == NULL) {  // still NULL?
+        if (_mi_is_heap_main(heap)) {
+          // the page info for the main heap is always allocated as part of an arena
+          arena_pages = &arena->pages_main;
+        }
+        else {
+          // always allocate the arena pages info from the main heap
+          // todo: allocate into the current arena?
+          arena_pages = mi_arena_pages_alloc(arena);
+        }
+        mi_atomic_store_ptr_release(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx], arena_pages);
+      }
+    }
+  }
+  if (_mi_is_heap_main(heap)) { mi_assert(arena_pages != NULL); }  // can never fail
+  return arena_pages;
+}
+
+static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slice_count, size_t block_size)
+{
+  mi_heap_t* const heap = _mi_theap_heap(theap);
+  const size_t tseq = theap->tld->thread_seq;
+  mi_arena_t* const req_arena = heap->exclusive_arena;
+
+  MI_UNUSED(slice_count);
+  const size_t bin = _mi_bin(block_size);
+  mi_assert_internal(bin < MI_BIN_COUNT);
+
+  // any abandoned in our size class?
+  mi_assert_internal(heap != NULL);
+  if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) {
+    return NULL;
+  }
+
+  // search arena's
+  const bool allow_large = true;
+  const int  any_numa = -1;
+  const bool match_numa = true;
+  mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, arena)
+  {
+    mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena);
+    if (arena_pages != NULL) {
+      size_t slice_index;
+      mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin];
+
+      if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) {
+        // found an abandoned page of the right size
+        // and claimed ownership.
+        mi_page_t* page = mi_arena_page_at_slice(arena, slice_index);
+        mi_assert_internal(mi_page_is_owned(page));
+        mi_assert_internal(mi_page_is_abandoned(page));
+        mi_assert_internal(mi_heap_has_page(heap, arena, page));
+        mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);
+        mi_theap_stat_decrease(theap, pages_abandoned, 1);
+        mi_theap_stat_counter_increase(theap, pages_reclaim_on_alloc, 1);
+
+        _mi_page_free_collect(page, false);  // update `used` count
+        mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
+        mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+        mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
+        mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+        mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+        mi_assert_internal(mi_page_block_size(page) == block_size);
+        mi_assert_internal(!mi_page_is_full(page));
+        return page;
+      }
+    }
+  }
+  mi_forall_suitable_arenas_end();
+  return NULL;
+}
+
+static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid) {
+  MI_UNUSED_RELEASE(block_size);
+  const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena page
+  const size_t page_alignment = MI_ARENA_SLICE_ALIGN;
+
+  mi_heap_t*  const heap = _mi_theap_heap(theap);
+  mi_tld_t*   const tld  = theap->tld;
+  mi_arena_t* const req_arena = heap->exclusive_arena;
+  const int numa_node = (heap->numa_node >= 0 ? heap->numa_node : tld->numa_node);
+
+  // try to allocate from free space in arena's
+  uint8_t* start = NULL;
+  *memid = _mi_memid_none();
+  const size_t alloc_size = mi_size_of_slices(slice_count);
+  if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) &&       // allowed to allocate from arena's?
+      !os_align &&                                                   // not large alignment
+      slice_count <= mi_arena_max_object_size()/MI_ARENA_SLICE_SIZE) // and not too large
+  {
+    start = (uint8_t*)mi_arenas_try_alloc(heap, slice_count, page_alignment, commit, allow_large, req_arena, tld->thread_seq, numa_node, memid);
+    if (start != NULL) {
+      mi_arena_pages_t* const arena_pages = mi_heap_ensure_arena_pages(heap, memid->mem.arena.arena);
+      if (arena_pages==NULL) {
+        _mi_arenas_free(start, mi_size_of_slices(slice_count), *memid); // roll back
+        start = NULL;
+      }
+      else {
+        // note: the following assert should hold if we could check it atomically, but in a concurrent setting we may already allocate in slice_count
+        // mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages, memid->mem.arena.slice_index, memid->mem.arena.slice_count));
+        mi_assert_internal(mi_bitmap_is_clear(arena_pages->pages, memid->mem.arena.slice_index));
+        mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index);
+      }
+    }
+  }
+
+  // otherwise fall back to the OS
+  if (start == NULL) {
+    if (os_align) {
+      // note: slice_count already includes the page
+      mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment));
+      start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid);
+    }
+    else {
+      start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid);
+    }
+  }
+
+  if (start == NULL) return NULL;
+  mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN));
+  mi_assert_internal(!os_align || _mi_is_aligned(start + page_alignment, block_alignment));
+  return start;
+}
+
+static size_t mi_page_block_start(size_t block_size, bool os_align)
+{
+  #if MI_GUARDED
+  // in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size
+  // this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages)
+  const size_t os_page_size = _mi_os_page_size();
+  mi_assert_internal(MI_PAGE_ALIGN >= os_page_size);
+  if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) {
+    return _mi_align_up(mi_page_info_size(), os_page_size);
+  }
+  else
+  #endif
+  if (os_align) {
+    return MI_PAGE_ALIGN;
+  }
+  else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {
+    // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB)
+    return _mi_align_up(mi_page_info_size(), block_size);
+  }
+  else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) {
+    // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB)
+    return _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2);
+  }
+  else {
+    // otherwise start after the info
+    return mi_page_info_size();
+  }
+}
+
+// Allocate a fresh page
+static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit)
+{
+  const bool os_align        = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
+  const size_t alloc_size    = mi_size_of_slices(slice_count);
+  mi_memid_t memid           = _mi_memid_none();
+  uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,block_size,block_alignment,os_align,commit,&memid);
+  if (!slice_start) return NULL;
+
+  // guard page at the end of mimalloc page?
+  #if MI_SECURE>=5
+  mi_assert(alloc_size > _mi_os_secure_guard_page_size());
+  const size_t page_noguard_size = alloc_size - _mi_os_secure_guard_page_size();
+  if (memid.initially_committed) {
+    _mi_os_secure_guard_page_set_at(slice_start + page_noguard_size, memid);
+  }
+  #else
+  const size_t page_noguard_size = alloc_size;
+  #endif
+
+  // allocate the page meta info
+  mi_page_t* page = NULL;
+  bool page_meta_is_separate = false;
+  size_t block_start = 0;
+
+  // allocate page meta info at the arena start?
+  if (memid.memkind == MI_MEM_ARENA) {
+    mi_arena_t* const arena = memid.mem.arena.arena;
+    if (arena->pages_meta != NULL) {
+      mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0);
+      mi_page_t* const page_meta = &arena->pages_meta[memid.mem.arena.slice_index];
+      mi_assert_internal(page_meta->block_size == 0);
+      #if MI_PAGE_META_ALIGNED_FREE_SMALL
+      // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice,
+      // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`)
+      if (block_size > MI_SMALL_SIZE_MAX)
+      #endif
+      {
+        page = page_meta;
+        page_meta_is_separate = true;
+        block_start = 0;
+        #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET)
+        #define MI_PAGE_BLOCK_START_MAX_OFFSET  (8*MI_INTPTR_BITS) /* 512 */
+        #endif
+        if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) {
+          block_start += block_size;
+        }
+        mi_assert_internal(page->block_size == 0);
+        _mi_memzero_aligned(page, sizeof(*page));
+      }
+    }
+  }
+  if (page == NULL) {
+    // put page meta info in front of the slice
+    page = (mi_page_t*)slice_start;
+    block_start = mi_page_block_start(block_size, os_align);
+  }
+
+  // commit first block?
+  size_t commit_size = 0;
+  if (!memid.initially_committed) {
+    commit_size = _mi_align_up(block_start + block_size, MI_PAGE_MIN_COMMIT_SIZE);
+    if (commit_size > page_noguard_size) { commit_size = page_noguard_size; }
+    bool is_zero = false;
+    if mi_unlikely(!mi_arena_commit( mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) {
+      _mi_arenas_free(slice_start, alloc_size, memid);
+      return NULL;
+    }
+  }
+  if (!memid.initially_zero && !page_meta_is_separate) {
+    _mi_memzero_aligned(page, sizeof(*page));
+  }
+
+  // claimed free slices: initialize the page partly
+  if (!memid.initially_zero && memid.initially_committed) {
+    mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);
+  }
+  else if (memid.initially_committed) {
+    mi_track_mem_defined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);
+  }
+  #if MI_DEBUG > 1
+  if (memid.initially_zero && memid.initially_committed) {
+    if (!mi_mem_is_zero(slice_start, page_noguard_size)) {
+      _mi_error_message(EFAULT, "internal error: page memory was not zero initialized.\n");
+      memid.initially_zero = false;
+      if (block_start > 0) { _mi_memzero_aligned(page, sizeof(*page)); }
+    }
+  }
+  #endif
+  const size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size);
+  mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX);
+
+  // initialize
+  page->reserved = (uint16_t)reserved;
+  page->page_start = slice_start + block_start;
+  page->block_size = block_size;
+  page->slice_committed = commit_size;
+  page->memid = memid;
+  page->free_is_zero = memid.initially_zero;
+  mi_assert_internal(page->free==NULL);
+  mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page));
+  mi_assert_internal(mi_page_slice_start(page) == slice_start);
+
+  // and own it
+  mi_page_claim_ownership(page);
+
+  // register in the page map
+  if mi_unlikely(!_mi_page_map_register(page)) {
+    _mi_arenas_free( slice_start, alloc_size, memid );
+    return NULL;
+  }
+
+  // stats
+  mi_theap_stat_increase(theap, pages, 1);
+  mi_theap_stat_increase(theap, page_bins[_mi_page_stats_bin(page)], 1);
+
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+  mi_assert_internal(mi_page_block_size(page) == block_size);
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(mi_page_is_owned(page));
+
+  return page;
+}
+
+// Allocate a regular small/medium/large page.
+static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_count, size_t block_size)
+{
+  // 1. look for an abandoned page
+  mi_page_t* page = mi_arenas_page_try_find_abandoned(theap, slice_count, block_size);
+  if (page != NULL) {
+    return page;  // return as abandoned
+  }
+
+  // 2. find a free block, potentially allocating a new arena
+  const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand);
+  const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) ||  // always commit small pages
+                       (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0));
+  page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit);
+  if (page == NULL) return NULL;
+
+  mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count);
+  if (!_mi_page_init(theap, page)) {
+    _mi_arenas_free( page, mi_page_full_size(page), page->memid);
+    return NULL;
+  }
+
+  return page;
+}
+
+// Allocate a page containing one block (very large, or with large alignment)
+static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment)
+{
+  const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
+  const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size());
+  #if MI_SECURE < 2
+  const size_t slice_count = mi_slice_count_of_size(info_size + block_size);
+  #else
+  const size_t slice_count = mi_slice_count_of_size(_mi_align_up(info_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size());
+  #endif
+
+  mi_page_t* page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, block_alignment, true /* commit singletons always */);
+  if (page == NULL) return NULL;
+
+  mi_assert(page->reserved == 1);
+  if (!_mi_page_init(theap, page)) {
+    _mi_arenas_free( page, mi_page_full_size(page), page->memid);
+    return NULL;
+  }
+
+  return page;
+}
+
+
+mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) {
+  mi_page_t* page;
+  if mi_unlikely(block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
+    mi_assert_internal(_mi_is_power_of_two(block_alignment));
+    page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);
+  }
+  else if (block_size <= MI_SMALL_MAX_OBJ_SIZE) {
+    page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_SMALL_PAGE_SIZE), block_size);
+  }
+  else if (block_size <= MI_MEDIUM_MAX_OBJ_SIZE) {
+    page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_MEDIUM_PAGE_SIZE), block_size);
+  }
+  #if MI_ENABLE_LARGE_PAGES
+  else if (block_size <= MI_LARGE_MAX_OBJ_SIZE) {
+    page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_LARGE_PAGE_SIZE), block_size);
+  }
+  #endif
+  else {
+    page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);
+  }
+  if mi_unlikely(page == NULL) {
+    return NULL;
+  }
+  // mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc == tld->subproc);
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+  mi_assert_internal(block_alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN || _mi_is_aligned(mi_page_start(page), block_alignment));
+
+  return page;
+}
+
+void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(mi_page_all_free(page));
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(page->next==NULL && page->prev==NULL);
+  mi_assert_internal(current_theapx == NULL || _mi_thread_id()==current_theapx->tld->thread_id);
+
+  if (current_theapx != NULL) {
+    mi_theap_stat_decrease(current_theapx, page_bins[_mi_page_stats_bin(page)], 1);
+    mi_theap_stat_decrease(current_theapx, pages, 1);
+  }
+  else {
+    mi_heap_t* const heap = mi_page_heap(page);
+    mi_heap_stat_decrease(heap, page_bins[_mi_page_stats_bin(page)], 1);
+    mi_heap_stat_decrease(heap, pages, 1);
+  }
+
+  #if MI_DEBUG>1
+  if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
+    size_t bin = _mi_bin(mi_page_block_size(page));
+    size_t slice_index;
+    size_t slice_count;
+    mi_arena_pages_t* arena_pages = NULL;
+    mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
+    mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
+    mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+    mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));
+    mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1));
+    // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may
+    // be (temporarily) not true if the free happens while trying to reclaim
+    // see `mi_arena_try_claim_abandoned`
+  }
+  #endif
+
+  // recommit guard page at the end?
+  // we must do this since we may later allocate large spans over this page and cannot have a guard page in between
+  #if MI_SECURE >= 5
+  if (!page->memid.is_pinned) {
+    _mi_os_secure_guard_page_reset_before(mi_page_slice_start(page) + mi_page_full_size(page), page->memid);
+  }
+  #endif
+
+  // unregister page
+  _mi_page_map_unregister(page);
+  if (page->memid.memkind == MI_MEM_ARENA) {
+    mi_arena_pages_t* arena_pages;
+    size_t slice_index;
+    size_t slice_count; MI_UNUSED(slice_count);
+    mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
+    mi_assert_internal(arena_pages!=NULL);
+    mi_bitmap_clear(arena_pages->pages, slice_index);
+    if (page->slice_committed > 0) {
+      // if committed on-demand, set the commit bits to account commit properly
+      mi_assert_internal(mi_page_full_size(page) >= page->slice_committed);
+      const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE;  // conservative
+      //mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices));
+      mi_assert_internal(slice_count >= total_slices);
+      if (total_slices > 0) {
+        mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL);
+      }
+      // any left over?
+      const size_t extra = page->slice_committed % MI_ARENA_SLICE_SIZE;
+      if (extra > 0) {
+        // pretend it was decommitted already
+        mi_subproc_stat_decrease(arena->subproc, committed, extra);
+      }
+    }
+    else {
+      mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+    }
+  }
+  if (mi_page_meta_is_separated(page)) { page->block_size = 0; }  // for assertion checking
+  _mi_arenas_free( mi_page_slice_start(page), mi_page_full_size(page), page->memid);
+}
+
+/* -----------------------------------------------------------
+  Arena abandon
+----------------------------------------------------------- */
+
+void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) {
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(!mi_page_all_free(page));
+  mi_assert_internal(page->next==NULL && page->prev == NULL);
+  mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);
+  // mi_assert_internal(current_theap == _mi_page_associated_theap(page));
+
+  mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(heap==_mi_theap_heap(current_theap));
+  if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
+    // make available for allocations
+    size_t bin = _mi_bin(mi_page_block_size(page));
+    size_t slice_index;
+    size_t slice_count;
+    mi_arena_pages_t* arena_pages = NULL;
+    mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);
+
+    mi_assert_internal(!mi_page_is_singleton(page));
+    mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
+    mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+    mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
+
+    mi_page_set_abandoned_mapped(page);
+    const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index);
+    MI_UNUSED(was_clear); mi_assert_internal(was_clear);
+    mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);
+    mi_theap_stat_increase(current_theap, pages_abandoned, 1);
+  }
+  else {
+    // page is full (or a singleton), or the page is OS/externally allocated
+    // leave as is; it will be reclaimed when an object is free'd in the page
+    // but for non-arena pages, add to the subproc list so these can be visited
+    if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {
+      mi_lock(&heap->os_abandoned_pages_lock) {
+        // push in front
+        page->prev = NULL;
+        page->next = heap->os_abandoned_pages;
+        if (page->next != NULL) { page->next->prev = page; }
+        heap->os_abandoned_pages = page;
+      }
+    }
+    mi_theap_stat_increase(current_theap, pages_abandoned, 1);
+  }
+  mi_abandoned_page_unown(page, current_theap);
+}
+
+
+// this is called from `free.c:mi_free_try_collect_mt` only.
+bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) {
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(!mi_page_is_abandoned_mapped(page));
+  mi_assert_internal(!mi_page_is_full(page));
+  mi_assert_internal(!mi_page_all_free(page));
+  mi_assert_internal(!mi_page_is_singleton(page));
+  if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) {
+    return false;
+  }
+  else {
+    // do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap
+    mi_theap_t* const theap = _mi_page_associated_theap_peek(page);
+    if (theap == NULL) {
+      return false;
+    }
+    else {
+      mi_theap_stat_counter_increase(theap, pages_reabandon_full, 1);
+      mi_theap_stat_adjust_decrease(theap, pages_abandoned, 1);  // adjust as we are not abandoning fresh
+      _mi_arenas_page_abandon(page, theap);
+      return true;
+    }
+  }
+}
+
+// called from `mi_free` if trying to unabandon an abandoned page
+void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(current_theapx==NULL || _mi_thread_id()==current_theapx->tld->thread_id);
+
+  mi_heap_t* const heap = mi_page_heap(page);
+  if (mi_page_is_abandoned_mapped(page)) {
+    mi_assert_internal(page->memid.memkind==MI_MEM_ARENA);
+    // remove from the abandoned map
+    size_t bin = _mi_bin(mi_page_block_size(page));
+    size_t slice_index;
+    size_t slice_count;
+    mi_arena_pages_t* arena_pages;
+    mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);  MI_UNUSED(arena);
+
+    mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
+    mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+
+    // this busy waits until a concurrent reader (from alloc_abandoned) is done
+    mi_bitmap_clear_once_set(arena_pages->pages_abandoned[bin], slice_index);
+    mi_page_clear_abandoned_mapped(page);
+    mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);
+  }
+  else {
+    // page is full (or a singleton), page is OS allocated
+    // if not an arena page, remove from the subproc os pages list
+    if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {
+      mi_lock(&heap->os_abandoned_pages_lock) {
+        if (page->prev != NULL) { page->prev->next = page->next; }
+        if (page->next != NULL) { page->next->prev = page->prev; }
+        if (heap->os_abandoned_pages == page) { heap->os_abandoned_pages = page->next; }
+        page->next = NULL;
+        page->prev = NULL;
+      }
+    }
+  }
+  if (current_theapx!=NULL) {
+    mi_theap_stat_decrease(current_theapx, pages_abandoned, 1);
+  }
+  else {
+    mi_heap_stat_decrease(heap, pages_abandoned, 1);
+  }
+}
+
+
+/* -----------------------------------------------------------
+  Arena free
+----------------------------------------------------------- */
+static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices);
+static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq);
+
+void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {
+  if (p==NULL) return;
+  if (size==0) return;
+
+  // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.)
+  mi_track_mem_undefined(p, size);
+
+  if (mi_memkind_is_os(memid.memkind)) {
+    // was a direct OS allocation, pass through
+    _mi_os_free(p, size, memid);
+  }
+  else if (memid.memkind == MI_MEM_ARENA) {
+    // allocated in an arena
+    size_t slice_count;
+    size_t slice_index;
+    mi_arena_t* arena = mi_arena_from_memid(memid, &slice_index, &slice_count);
+    mi_assert_internal((size%MI_ARENA_SLICE_SIZE)==0);
+    mi_assert_internal((slice_count*MI_ARENA_SLICE_SIZE)==size);
+    mi_assert_internal(mi_arena_slice_start(arena,slice_index) <= (uint8_t*)p);
+    mi_assert_internal(mi_arena_slice_start(arena,slice_index) + mi_size_of_slices(slice_count) > (uint8_t*)p);
+    // checks
+    if (arena == NULL) {
+      _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memid: 0x%zx\n", p, size, memid);
+      return;
+    }
+    mi_assert_internal(slice_index < arena->slice_count);
+    mi_assert_internal(slice_index >= mi_arena_info_slices(arena));
+    if (slice_index < mi_arena_info_slices(arena) || slice_index >= arena->slice_count) {
+      _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid);
+      return;
+    }
+
+    // potentially decommit
+    if (!arena->memid.is_pinned /* && !arena->memid.initially_committed */) { // todo: allow decommit even if initially committed?
+      // (delay) purge the page
+      mi_arena_schedule_purge(arena, slice_index, slice_count);
+    }
+
+    // and make it available to others again
+    bool all_inuse = mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);
+    if (!all_inuse) {
+      _mi_error_message(EAGAIN, "trying to free an already freed arena block: %p, size %zu\n", mi_arena_slice_start(arena,slice_index), mi_size_of_slices(slice_count));
+      return;
+    };
+  }
+  else if (memid.memkind == MI_MEM_META) {
+    _mi_meta_free(p, size, memid);
+  }
+  else {
+    // arena was none, external, or static; nothing to do
+    mi_assert_internal(mi_memid_needs_no_free(memid));
+  }
+
+  // try to purge expired decommits
+  // mi_arenas_try_purge(false, false, NULL);
+}
+
+// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired
+void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld) {
+  mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq);
+}
+
+
+// Is a pointer contained in the given arena area?
+static bool mi_arena_strictly_contains(mi_arena_t* arena, const void* p) {
+  return (arena != NULL &&
+          mi_arena_start(arena) <= (const uint8_t*)p &&
+          mi_arena_start(arena) + mi_size_of_slices(arena->slice_count) >(const uint8_t*)p);
+}
+
+// Is a pointer inside any of our arenas?
+static bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) {
+  mi_subproc_t* subproc = _mi_subproc();
+  const size_t max_arena = mi_arenas_get_count(subproc);
+  for (size_t i = 0; i < max_arena; i++) {
+    mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
+    if (arena != NULL) {
+      if (parent==NULL || arena==parent || arena->parent==parent) {
+        if (mi_arena_strictly_contains(arena, p)) {
+          return true;
+        }
+      }
+    }
+  }
+  return false;
+}
+
+// Is a pointer inside any of our arenas?
+bool _mi_arenas_contain(const void* p) {
+  return mi_arenas_contain_ex(p, NULL);
+}
+
+// Is a pointer contained in the given arena area?
+bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) {
+  mi_arena_t* arena = _mi_arena_from_id(arena_id);
+  if (arena==NULL) return false;
+  else if (mi_arena_strictly_contains(arena, p)) return true;
+  else return mi_arenas_contain_ex(p, arena);  // maybe a subarena?
+}
+
+
+/* -----------------------------------------------------------
+  Remove an arena.
+----------------------------------------------------------- */
+
+// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`
+// for dynamic libraries that are unloaded and need to release all their allocated memory.
+static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) {
+  mi_assert_internal(subproc != NULL);
+  const size_t arena_count = mi_arenas_get_count(subproc);
+  for (size_t i = 0; i < arena_count; i++) {
+    mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
+    if (arena != NULL) {
+      // mi_lock_done(&arena->abandoned_visit_lock);
+      mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);
+      if (mi_memkind_is_os(arena->memid.memkind)) {
+        _mi_os_free_ex(mi_arena_start(arena), mi_arena_size(arena), true, arena->memid, subproc); // pass `subproc` to avoid accessing the theap pointer (in `_mi_subproc()`)
+      }
+    }
+  }
+  // try to lower the max arena.
+  size_t expected = arena_count;
+  mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, (size_t)0);
+}
+
+
+// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`
+// for dynamic libraries that are unloaded and need to release all their allocated memory.
+void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc) {
+  mi_arenas_unsafe_destroy(subproc);
+  // mi_arenas_try_purge(true /* force purge */, true /* visit all*/, subproc, 0 /* thread seq */);  // purge non-owned arenas
+}
+
+
+/* -----------------------------------------------------------
+  Add an arena.
+----------------------------------------------------------- */
+
+static bool mi_arenas_add(mi_subproc_t* subproc, mi_arena_t* arena, mi_arena_id_t* arena_id)
+{
+  mi_assert_internal(arena != NULL);
+  mi_assert_internal(arena->slice_count > 0);
+  if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }
+
+  // try to find a NULL entry
+  mi_arena_t* expected;
+  size_t count = mi_arenas_get_count(subproc);
+  for( size_t i = 0; i < count; i++) {
+    if (mi_arena_from_index(subproc,i) == NULL) {
+      arena->arena_idx = i;
+      expected = NULL;
+      if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[i], &expected, arena)) {
+        // success
+        if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); }
+        return true;
+      }
+    }
+  }
+
+  // otherwise, try to allocate a fresh slot
+  while(count<MI_MAX_ARENAS) {
+    if (mi_atomic_cas_strong_release(&subproc->arena_count, &count, count+1)) {
+      arena->arena_idx = count;
+      expected = NULL;
+      if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[count], &expected, arena)) {
+        mi_subproc_stat_counter_increase(arena->subproc, arena_count, 1);
+        if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); }
+        return true;
+      }
+    }
+  }
+
+  // failed
+  arena->arena_idx = 0;
+  arena->subproc = NULL;
+  return false;
+}
+
+static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) {
+  if (slice_count == 0) slice_count = MI_BCHUNK_BITS;
+  mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);
+  const size_t base_size = _mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE);
+  const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned
+  const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL);
+  const size_t size = base_size + bitmaps_size;
+  if (bitmap_base != NULL) *bitmap_base = base_size;
+  return size;
+}
+
+static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) {
+  if (slice_count == 0) slice_count = MI_BCHUNK_BITS;
+  mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);
+  const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE);
+  const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned
+  const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free
+  #if MI_PAGE_META_IS_SEPARATED
+  const size_t pages_size = slice_count * sizeof(mi_page_t);
+  #else
+  const size_t pages_size = 0;
+  #endif
+  const size_t size = base_size + bitmaps_size + pages_size;
+
+  const size_t os_page_size = _mi_os_page_size();
+  const size_t info_size = _mi_align_up(size, os_page_size) + _mi_os_secure_guard_page_size();
+  const size_t info_slices = mi_slice_count_of_size(info_size);
+
+  if (bitmap_base != NULL) *bitmap_base = base_size;
+  return info_slices;
+}
+
+static mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) {
+  mi_bitmap_t* bitmap = (mi_bitmap_t*)(*base);
+  *base = (*base) + mi_bitmap_init(bitmap, slice_count, true /* already zero */);
+  return bitmap;
+}
+
+static mi_bbitmap_t* mi_arena_bbitmap_init(size_t slice_count, uint8_t** base) {
+  mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base);
+  *base = (*base) + mi_bbitmap_init(bbitmap, slice_count, true /* already zero */);
+  return bbitmap;
+}
+
+static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) {
+  const size_t slice_count = arena->slice_count;
+  size_t bitmap_base = 0;
+  const size_t size = mi_arena_pages_size(slice_count, &bitmap_base);
+  mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(mi_heap_main(), size, MI_BCHUNK_SIZE);
+  if (arena_pages==NULL) return NULL;
+  uint8_t* base = (uint8_t*)arena_pages + bitmap_base;
+  mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE));
+  arena_pages->pages = mi_arena_bitmap_init(slice_count, &base);
+  for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
+    arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);
+  }
+  return arena_pages;
+}
+
+static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
+                                        size_t slice_count, mi_arena_t* parent, size_t total_size,
+                                        int numa_node, bool exclusive,
+                                        mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id)
+{
+  mi_assert_internal(_mi_is_aligned(start,MI_ARENA_SLICE_ALIGN));
+  mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE);
+
+  if (slice_count > MI_BITMAP_MAX_BIT_COUNT) {  // 16 GiB for now
+    // note: this should never happen if called from `mi_manage_os_memory` (as that allocates sub-arenas when needed)
+    _mi_warning_message("cannot use OS memory since it is too large (size %zu MiB, maximum is %zu MiB)", mi_size_of_slices(slice_count)/MI_MiB, mi_size_of_slices(MI_BITMAP_MAX_BIT_COUNT)/MI_MiB);
+    return NULL;
+  }
+
+  size_t bitmap_base;
+  const size_t info_slices = mi_arena_info_slices_needed(slice_count, &bitmap_base);
+  if (slice_count < info_slices+1) {
+    _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", mi_size_of_slices(slice_count)/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB);
+    return NULL;
+  }
+  else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) {
+    _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES);
+    return NULL;
+  }
+
+  mi_arena_t* arena = (mi_arena_t*)start;
+
+  // commit & zero if needed
+  if (!memid.initially_committed) {
+    size_t commit_size = mi_size_of_slices(info_slices);
+    // leave a guard OS page decommitted at the end?
+    if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); }
+    bool ok = false;
+    if (commit_fun != NULL) {
+      ok = (*commit_fun)(true /* commit */, arena, commit_size, NULL, commit_fun_arg);
+    }
+    else {
+      ok = _mi_os_commit(arena, commit_size, NULL);
+    }
+    if (!ok) {
+      _mi_warning_message("unable to commit meta-data for OS memory");
+      return NULL;
+    }
+  }
+  else if (!memid.is_pinned) {
+    // if MI_SECURE, set a guard page at the end of the arena info
+    // todo: this does not respect the commit_fun as the memid is of external memory
+    _mi_os_secure_guard_page_set_before((uint8_t*)arena + mi_size_of_slices(info_slices), memid);
+  }
+  if (!memid.initially_zero) {
+    _mi_memzero(arena, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size());
+  }
+
+  // init
+  arena->subproc = subproc;
+  arena->memid = memid;
+  arena->is_exclusive = exclusive;
+  arena->slice_count = slice_count;
+  arena->info_slices = info_slices;
+  if (numa_node<0 && mi_option_is_enabled(mi_option_arena_is_numa_local)) {
+    arena->numa_node = _mi_os_numa_node();
+  }
+  else {
+    arena->numa_node = numa_node;
+  }
+  arena->purge_expire = 0;
+  arena->commit_fun = commit_fun;
+  arena->commit_fun_arg = commit_fun_arg;
+  arena->parent = parent;
+  arena->total_size = total_size;
+
+  // init bitmaps
+  uint8_t* base = mi_arena_start(arena) + bitmap_base;
+  arena->slices_free = mi_arena_bbitmap_init(slice_count, &base);
+  arena->slices_committed = mi_arena_bitmap_init(slice_count, &base);
+  arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base);
+  arena->slices_purge = mi_arena_bitmap_init(slice_count, &base);
+  arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base);
+  for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
+    arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);
+  }
+  #if MI_PAGE_META_IS_SEPARATED
+  arena->pages_meta = (mi_page_t*)base;
+  base += (slice_count * sizeof(mi_page_t));
+  #else
+  arena->pages_meta = NULL;
+  #endif
+  mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena)));
+
+  // reserve our meta info (and reserve slices outside the memory area)
+  mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices);
+  if (memid.initially_committed) {
+    mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count);
+  }
+  if (!memid.initially_zero) {
+    mi_bitmap_unsafe_setN(arena->slices_dirty, 0, arena->slice_count);
+  }
+
+  if (!mi_arenas_add(subproc, arena, arena_id)) { return NULL;  }
+  return arena;
+}
+
+static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t size, int numa_node, bool exclusive,
+  mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept
+{
+  // checks
+  mi_assert(_mi_is_aligned(start, MI_ARENA_SLICE_SIZE));
+  mi_assert(start!=NULL);
+  if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }
+  if (start==NULL) return false;
+  if (!_mi_is_aligned(start, MI_ARENA_SLICE_SIZE)) {
+    // we can align the start since the memid tracks the real base of the memory.
+    void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_SLICE_SIZE);
+    const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start;
+    if (diff >= size || (size - diff) < MI_ARENA_SLICE_SIZE) {
+      _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size);
+      return false;
+    }
+    start = aligned_start;
+    size = size - diff;
+  }
+
+  // allocate enough arena's to span the full memory area
+  // the first arena is the owner, the rest are "sub-arena" (with `parent` pointing to the first one)
+  size_t total_slice_count = _mi_align_down(size / MI_ARENA_SLICE_SIZE, MI_BCHUNK_BITS);
+  size_t total_size = mi_size_of_slices(total_slice_count);
+  if (total_size < MI_ARENA_MIN_SIZE) {
+    _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", size/MI_KiB, MI_ARENA_MIN_SIZE/MI_KiB);
+    return false;
+  }
+
+  mi_arena_t* parent = NULL;
+  do {
+    // counting down on the total_slice_count
+    size_t slice_count = total_slice_count;
+    if (slice_count > MI_BITMAP_MAX_BIT_COUNT) {  // 16 GiB for now (with 64KiB slices)
+      slice_count = MI_BITMAP_MAX_BIT_COUNT;
+    }
+
+    // initialize
+    mi_arena_t* arena = mi_arena_initialize( subproc, start, slice_count, parent,
+                                              (parent==NULL ? total_size : 0), numa_node, exclusive,
+                                              memid, commit_fun, commit_fun_arg,
+                                              (parent==NULL ? arena_id : NULL));
+    if (arena==NULL) {
+      // failed to initialize due to failing commit or too many arena's
+      if (parent==NULL) {
+        return false;
+      }
+      else {
+        // partial success, but failed to use the full area..
+        // todo: roll-back in this case? that requires a lock on the arena's array though
+        mi_assert(mi_size_of_slices(total_slice_count) <= parent->total_size);
+        parent->total_size -= mi_size_of_slices(total_slice_count);
+        return true;
+      }
+    }
+
+    // success
+    if (parent==NULL) {
+      parent = arena;
+      memid.memkind = MI_MEM_NONE;
+    }
+    mi_assert(slice_count <= total_slice_count);
+    total_slice_count -= slice_count;
+    start = (uint8_t*)start + mi_size_of_slices(slice_count);
+  }
+  while (total_slice_count > 0);
+
+  return true;
+}
+
+bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {
+  mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL);
+  memid.mem.os.base = start;
+  memid.mem.os.size = size;
+  memid.initially_committed = is_committed;
+  memid.initially_zero = is_zero;
+  memid.is_pinned = is_pinned;
+  return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, NULL, NULL, arena_id);
+}
+
+bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept
+{
+  mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL);
+  memid.mem.os.base = start;
+  memid.mem.os.size = size;
+  memid.initially_committed = is_committed;
+  memid.initially_zero = is_zero;
+  memid.is_pinned = is_pinned;
+  return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, commit_fun, commit_fun_arg, arena_id);
+}
+
+
+// Reserve a range of regular OS memory
+static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) {
+  if (arena_id != NULL) *arena_id = _mi_arena_id_none();
+  size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice
+  mi_memid_t memid;
+  void* start = _mi_os_alloc_aligned(size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid);
+  if (start == NULL) return ENOMEM;
+  if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) {
+    _mi_os_free_ex(start, size, commit, memid, NULL);
+    _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024));
+    return ENOMEM;
+  }
+  _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size, 1024), memid.is_pinned ? " (in large os pages)" : "");
+  // mi_debug_show_arenas(true, true, false);
+
+  return 0;
+}
+
+// Reserve a range of regular OS memory
+int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {
+  return mi_reserve_os_memory_ex2(_mi_subproc(), size, commit, allow_large, exclusive, arena_id);
+}
+
+// Manage a range of regular OS memory
+bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept {
+  return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL);
+}
+
+// Reserve a range of regular OS memory
+int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept {
+  return mi_reserve_os_memory_ex(size, commit, allow_large, false, NULL);
+}
+
+
+/* -----------------------------------------------------------
+  Debugging
+----------------------------------------------------------- */
+
+// Return idx of the slice past the last used slice
+static size_t mi_arena_used_slices(mi_arena_t* arena) {
+  size_t idx;
+  if (mi_bbitmap_bsr_inv(arena->slices_free, &idx)) {
+    return (idx + 1);
+  }
+  else {
+    return mi_arena_info_slices(arena);
+  }
+}
+
+static size_t mi_debug_show_bfield(mi_bfield_t field, char* buf, size_t* k) {
+  size_t bit_set_count = 0;
+  for (int bit = 0; bit < MI_BFIELD_BITS; bit++) {
+    bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0);
+    if (is_set) bit_set_count++;
+    buf[*k] = (is_set ? 'x' : '.');
+    *k = *k + 1;
+  }
+  return bit_set_count;
+}
+
+typedef enum mi_ansi_color_e {
+  MI_BLACK = 30,
+  MI_MAROON,
+  MI_DARKGREEN,
+  MI_ORANGE,
+  MI_NAVY,
+  MI_PURPLE,
+  MI_TEAL,
+  MI_GRAY,
+  MI_DARKGRAY = 90,
+  MI_RED,
+  MI_GREEN,
+  MI_YELLOW,
+  MI_BLUE,
+  MI_MAGENTA,
+  MI_CYAN,
+  MI_WHITE
+} mi_ansi_color_t;
+
+static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) {
+  *k += _mi_snprintf(buf + *k, 32, "\x1B[%dm", (int)color);
+}
+
+static int mi_page_commit_usage(mi_page_t* page) {
+  // if (mi_page_size(page) <= MI_PAGE_MIN_COMMIT_SIZE) return 100;
+  const size_t committed_size = mi_page_committed(page);
+  const size_t used_size = page->used * mi_page_block_size(page);
+  return (int)(used_size * 100 / committed_size);
+}
+
+static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, size_t slice_index, long* pbit_of_page, mi_ansi_color_t* pcolor_of_page ) {
+  size_t bit_set_count = 0;
+  long bit_of_page = *pbit_of_page;
+  mi_ansi_color_t color = *pcolor_of_page;
+  mi_ansi_color_t prev_color = MI_GRAY;
+  for (int bit = 0; bit < MI_BFIELD_BITS; bit++, bit_of_page--) {
+    // bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0);
+    void* start = mi_arena_slice_start(arena, slice_index + bit);
+    mi_page_t* page = _mi_safe_ptr_page(start);
+    char c = ' ';
+    if (page!=NULL && start==mi_page_slice_start(page)) {
+      mi_assert_internal(bit_of_page <= 0);
+      bit_set_count++;
+      c = 'p';
+      color = MI_GRAY;
+      if (mi_page_is_singleton(page)) { c = 's'; }
+      else if (mi_page_is_full(page)) { c = 'f'; }
+      if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); }
+      int commit_usage = mi_page_commit_usage(page);
+      if (commit_usage < 25) { color = MI_MAROON; }
+      else if (commit_usage < 50) { color = MI_ORANGE; }
+      else if (commit_usage < 75) { color = MI_TEAL; }
+      else color = MI_DARKGREEN;
+      bit_of_page = (long)page->memid.mem.arena.slice_count;
+    }
+    else {
+      c = '?';
+      if (bit_of_page > 0) { c = '-'; }
+      else if (_mi_meta_is_meta_page(start)) { c = 'm'; color = MI_GRAY; }
+      else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; }
+      // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; }
+      else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) {
+        if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; }
+        else if (mi_bitmap_is_set(arena->slices_committed, slice_index + bit)) { c = '_'; color = MI_GRAY; }
+        else { c = '.'; color = MI_GRAY; }
+      }
+      if (bit==MI_BFIELD_BITS-1 && bit_of_page > 1) { c = '>'; }
+    }
+    if (color != prev_color) {
+      mi_debug_color(buf, k, color);
+      prev_color = color;
+    }
+    buf[*k] = c; *k += 1;
+  }
+  mi_debug_color(buf, k, MI_GRAY);
+  *pbit_of_page = bit_of_page;
+  *pcolor_of_page = color;
+  return bit_set_count;
+}
+
+static size_t mi_debug_show_chunks(const char* header1, const char* header2, const char* header3,
+                                   size_t slice_count, size_t chunk_count,
+                                   mi_bchunk_t* chunks, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow)
+{
+  _mi_raw_message("\x1B[37m%s%s%s (use/commit: \x1B[31m0 - 25%%\x1B[33m - 50%%\x1B[36m - 75%%\x1B[32m - 100%%\x1B[0m)\n", header1, header2, header3);
+  const size_t fields_per_line = (narrow ? 2 : 4);
+  const size_t used_slice_count = mi_arena_used_slices(arena);
+  size_t bit_count = 0;
+  size_t bit_set_count = 0;
+  long bit_of_page = 0;
+  mi_ansi_color_t color_of_page = MI_GRAY;
+  for (size_t i = 0; i < chunk_count && bit_count < slice_count; i++) {
+    char buf[5*MI_BCHUNK_BITS + 64]; _mi_memzero(buf, sizeof(buf));
+    if (bit_count > used_slice_count && i+2 < chunk_count) {
+      const size_t diff = chunk_count - 1 - i;
+      bit_count += diff*MI_BCHUNK_BITS;
+      _mi_raw_message("  |\n");
+      i = chunk_count-1;
+    }
+
+    size_t k = 0;
+
+    if (i<10)        { buf[k++] = ('0' + (char)i); buf[k++] = ' '; buf[k++] = ' '; }
+    else if (i<100)  { buf[k++] = ('0' + (char)(i/10)); buf[k++] = ('0' + (char)(i%10)); buf[k++] = ' '; }
+    else if (i<1000) { buf[k++] = ('0' + (char)(i/100)); buf[k++] = ('0' + (char)((i%100)/10)); buf[k++] = ('0' + (char)(i%10)); }
+
+    char chunk_kind = ' ';
+    if (chunk_bins != NULL) {
+      switch (mi_bbitmap_debug_get_bin(chunk_bins,i)) {
+        case MI_CBIN_SMALL:  chunk_kind = 'S'; break;
+        case MI_CBIN_MEDIUM: chunk_kind = 'M'; break;
+        case MI_CBIN_LARGE:  chunk_kind = 'L'; break;
+        case MI_CBIN_HUGE:   chunk_kind = 'H'; break;
+        case MI_CBIN_OTHER:  chunk_kind = 'X'; break;
+        default: chunk_kind = ' '; break; // suppress warning
+        // case MI_CBIN_NONE: chunk_kind = 'N'; break;
+      }
+    }
+    buf[k++] = chunk_kind;
+    buf[k++] = ' ';
+
+    for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
+      if (j > 0 && (j % fields_per_line) == 0) {
+        // buf[k++] = '\n'; _mi_memset(buf+k,' ',7); k += 7;
+        _mi_raw_message("  %s\n\x1B[37m", buf);
+        _mi_memzero(buf, sizeof(buf));
+        _mi_memset(buf, ' ', 5); k = 5;
+      }
+      if (bit_count < slice_count) {
+        mi_bfield_t bfield = 0;
+        if (chunks!=NULL) {
+          bfield = chunks[i].bfields[j];
+        }
+        if (invert) bfield = ~bfield;
+        size_t xcount = (chunks==NULL ? mi_debug_show_page_bfield(buf, &k, arena, bit_count, &bit_of_page, &color_of_page)
+                                      : mi_debug_show_bfield(bfield, buf, &k));
+        if (invert) xcount = MI_BFIELD_BITS - xcount;
+        bit_set_count += xcount;
+        buf[k++] = ' ';
+      }
+      else {
+        _mi_memset(buf + k, 'o', MI_BFIELD_BITS);
+        k += MI_BFIELD_BITS;
+      }
+      bit_count += MI_BFIELD_BITS;
+    }
+    _mi_raw_message("  %s\n\x1B[37m", buf);
+  }
+  _mi_raw_message("\x1B[0m  total pages: %zu\n", bit_set_count);
+  return bit_set_count;
+}
+
+//static size_t mi_debug_show_bitmap_binned(const char* header1, const char* header2, const char* header3, size_t slice_count,
+//                                           mi_bitmap_t* bitmap, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow) {
+//  return mi_debug_show_chunks(header1, header2, header3, slice_count, mi_bitmap_chunk_count(bitmap), &bitmap->chunks[0], chunk_bins, invert, arena, narrow);
+//}
+
+static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narrow) mi_attr_noexcept {
+  mi_subproc_t* subproc = heap->subproc;
+  size_t max_arenas = mi_arenas_get_count(subproc);
+  //size_t free_total = 0;
+  //size_t slice_total = 0;
+  //size_t abandoned_total = 0;
+  size_t page_total = 0;
+  for (size_t i = 0; i < max_arenas; i++) {
+    mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
+    if (arena == NULL) break;
+    mi_assert(arena->subproc == subproc);
+    // slice_total += arena->slice_count;
+    _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %p, numa: %i\n",
+        (arena->parent==NULL ? "" : "(sub)"), i, arena, arena->slice_count, (size_t)(mi_size_of_slices(arena->slice_count)/MI_MiB),
+        (arena->memid.is_pinned ? ", pinned" : ""), (arena->is_exclusive ? ", exclusive" : ""),
+        arena->subproc, arena->numa_node);
+    //if (show_inuse) {
+    //  free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL);
+    //}
+    //if (show_committed) {
+    //  mi_debug_show_bitmap("committed slices", arena->slice_count, arena->slices_committed, false, NULL);
+    //}
+    // todo: abandoned slices
+    //if (show_purge) {
+    //  purge_total += mi_debug_show_bitmap("purgeable slices", arena->slice_count, arena->slices_purge, false, NULL);
+    //}
+    if (show_pages) {
+      // mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);
+      // if (arena_pages != NULL)
+      {
+        const char* header1 = "chunks (p:page, f:full, s:singleton, P,F,S:not abandoned, i:arena-info, m:meta-data, ~:free-purgable, _:free-committed, .:free-reserved)";
+        const char* header2 = (narrow ? "\n       " : " ");
+        const char* header3 = "(chunk bin: S:small, M : medium, L : large, X : other)";
+        page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count,
+                                           mi_bbitmap_chunk_count(arena->slices_free), NULL,
+                                           arena->slices_free->chunkmap_bins, false, arena, narrow);
+      }
+    }
+  }
+  // if (show_inuse)     _mi_raw_message("total inuse slices    : %zu\n", slice_total - free_total);
+  // if (show_abandoned) _mi_raw_message("total abandoned slices: %zu\n", abandoned_total);
+  if (show_pages) _mi_raw_message("total pages in arenas: %zu\n", page_total);
+}
+
+void mi_debug_show_arenas(void) mi_attr_noexcept {
+  mi_debug_show_arenas_ex(mi_heap_main(), true /* show pages */, true /* narrow? */);
+}
+
+void mi_arenas_print(void) mi_attr_noexcept {
+  mi_debug_show_arenas();
+}
+
+
+/* -----------------------------------------------------------
+  Reserve a huge page arena.
+----------------------------------------------------------- */
+// reserve at a specific numa node
+int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {
+  if (arena_id != NULL) *arena_id = NULL;
+  if (pages==0) return 0;
+  if (numa_node < -1) numa_node = -1;
+  if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count();
+  size_t hsize = 0;
+  size_t pages_reserved = 0;
+  mi_memid_t memid;
+  void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid);
+  if (p==NULL || pages_reserved==0) {
+    _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages);
+    return ENOMEM;
+  }
+  _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages);
+
+  if (!mi_manage_os_memory_ex2(_mi_subproc(), p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) {
+    _mi_os_free(p, hsize, memid);
+    return ENOMEM;
+  }
+  return 0;
+}
+
+int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept {
+  return mi_reserve_huge_os_pages_at_ex(pages, numa_node, timeout_msecs, false, NULL);
+}
+
+// reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)
+int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept {
+  if (pages == 0) return 0;
+
+  // pages per numa node
+  int numa_count = (numa_nodes > 0 && numa_nodes <= INT_MAX ? (int)numa_nodes : _mi_os_numa_node_count());
+  if (numa_count <= 0) { numa_count = 1; }
+  const size_t pages_per = pages / numa_count;
+  const size_t pages_mod = pages % numa_count;
+  const size_t timeout_per = (timeout_msecs==0 ? 0 : (timeout_msecs / numa_count) + 50);
+
+  // reserve evenly among numa nodes
+  for (int numa_node = 0; numa_node < numa_count && pages > 0; numa_node++) {
+    size_t node_pages = pages_per;  // can be 0
+    if ((size_t)numa_node < pages_mod) { node_pages++; }
+    int err = mi_reserve_huge_os_pages_at(node_pages, numa_node, timeout_per);
+    if (err) return err;
+    if (pages < node_pages) {
+      pages = 0;
+    }
+    else {
+      pages -= node_pages;
+    }
+  }
+
+  return 0;
+}
+
+int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept {
+  MI_UNUSED(max_secs);
+  _mi_warning_message("mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\n");
+  if (pages_reserved != NULL) *pages_reserved = 0;
+  int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0));
+  if (err==0 && pages_reserved!=NULL) *pages_reserved = pages;
+  return err;
+}
+
+
+
+
+
+/* -----------------------------------------------------------
+  Arena purge
+----------------------------------------------------------- */
+
+static long mi_arena_purge_delay(void) {
+  // <0 = no purging allowed, 0=immediate purging, >0=milli-second delay
+  return (mi_option_get(mi_option_purge_delay) * mi_option_get(mi_option_arena_purge_mult));
+}
+
+// reset or decommit in an arena and update the commit bitmap
+// assumes we own the area (i.e. slices_free is claimed by us)
+// returns if the memory is no longer committed (versus reset which keeps the commit)
+static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) {
+  mi_assert_internal(!arena->memid.is_pinned);
+  mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
+
+  const size_t size = mi_size_of_slices(slice_count);
+  void* const p = mi_arena_slice_start(arena, slice_index);
+  //const bool all_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);
+  size_t already_committed;
+  mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..)
+  const bool all_committed = (already_committed == slice_count);
+  const bool needs_recommit = _mi_os_purge_ex(p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg);
+
+  if (needs_recommit) {
+    // no longer committed
+    mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
+    // we just counted in the purge to decommit all, but the some part was not committed so adjust that here
+    // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(slice_count - already_committed));
+  }
+  else if (!all_committed) {
+    // we cannot assume any of these are committed any longer (even with reset since we did setN and may have marked uncommitted slices as committed)
+    mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
+    // we adjust the commit count as parts will be re-committed
+    // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed));
+  }
+
+  return needs_recommit;
+}
+
+
+// Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls.
+// Note: assumes we (still) own the area as we may purge immediately
+static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) {
+  const long delay = mi_arena_purge_delay();
+  if (arena->memid.is_pinned || delay < 0 || _mi_preloading()) return;  // is purging allowed at all?
+
+  mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
+  if (delay == 0) {
+    // purge directly
+    mi_arena_purge(arena, slice_index, slice_count);
+  }
+  else {
+    // schedule purge
+    const mi_msecs_t expire = _mi_clock_now() + delay;
+    mi_msecs_t expire0 = 0;
+    if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) {
+      // expiration was not yet set
+      // maybe set the global arenas expire as well (if it wasn't set already)
+      mi_assert_internal(expire0==0);
+      mi_atomic_casi64_strong_acq_rel(&arena->subproc->purge_expire, &expire0, expire);
+    }
+    else {
+      // already an expiration was set
+    }
+    mi_bitmap_setN(arena->slices_purge, slice_index, slice_count, NULL);
+  }
+}
+
+typedef struct mi_purge_visit_info_s {
+  mi_msecs_t now;
+  mi_msecs_t delay;
+  bool all_purged;
+  bool any_purged;
+} mi_purge_visit_info_t;
+
+static bool mi_arena_try_purge_range(mi_arena_t* arena, size_t slice_index, size_t slice_count) {
+  mi_assert(slice_count < MI_BCHUNK_BITS);
+  if (mi_bbitmap_try_clearNC(arena->slices_free, slice_index, slice_count)) {
+    // purge
+    bool decommitted = mi_arena_purge(arena, slice_index, slice_count); MI_UNUSED(decommitted);
+    mi_assert_internal(!decommitted || mi_bitmap_is_clearN(arena->slices_committed, slice_index, slice_count));
+    // and reset the free range
+    mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);
+    return true;
+  }
+  else {
+    // was allocated again already
+    return false;
+  }
+}
+
+static bool mi_arena_try_purge_visitor(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) {
+  mi_purge_visit_info_t* vinfo = (mi_purge_visit_info_t*)arg;
+  // try to purge: first claim the free blocks
+  if (mi_arena_try_purge_range(arena, slice_index, slice_count)) {
+    vinfo->any_purged = true;
+    vinfo->all_purged = true;
+  }
+  else if (slice_count > 1)
+  {
+    // failed to claim the full range, try per slice instead
+    for (size_t i = 0; i < slice_count; i++) {
+      const bool purged = mi_arena_try_purge_range(arena, slice_index + i, 1);
+      vinfo->any_purged = vinfo->any_purged || purged;
+      vinfo->all_purged = vinfo->all_purged && purged;
+    }
+  }
+  // don't clear the purge bits as that is done atomically be the _bitmap_forall_set_ranges
+  // mi_bitmap_clearN(arena->slices_purge, slice_index, slice_count);
+  return true; // continue
+}
+
+// returns
+// -1 = nothing was purged
+// 0  = nothing was purged yet because have not yet reached the expire time
+// 1  = some pages in the arena were purged
+static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
+{
+  // check pre-conditions
+  if (arena->memid.is_pinned) return -1;
+
+  // expired yet?
+  mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);
+  if (!force) {
+    if (expire==0) return -1;
+    if (expire > now) return 0;
+  }
+
+  // reset expire
+  mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0);
+  mi_subproc_stat_counter_increase(arena->subproc, arena_purges, 1);
+
+  // go through all purge info's  (with max MI_BFIELD_BITS ranges at a time)
+  // this also clears those ranges atomically (so any newly freed blocks will get purged next
+  // time around)
+  mi_purge_visit_info_t vinfo = { now, mi_arena_purge_delay(), true /*all?*/, false /*any?*/};
+
+  // we purge by at least `minslices` to not fragment transparent huge pages for example
+  const size_t minslices = mi_slice_count_of_size(_mi_os_minimal_purge_size());
+  _mi_bitmap_forall_setc_rangesn(arena->slices_purge, minslices, &mi_arena_try_purge_visitor, arena, &vinfo);
+
+  return (vinfo.any_purged ? 1 : -1);
+}
+
+
+static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq)
+{
+  // try purge can be called often so try to only run when needed
+  const long delay = mi_arena_purge_delay();
+  if (_mi_preloading() || delay <= 0) return;  // nothing will be scheduled
+
+  // check if any arena needs purging?
+  const mi_msecs_t now = _mi_clock_now();
+  const mi_msecs_t arenas_expire = mi_atomic_loadi64_acquire(&subproc->purge_expire);
+  if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return;
+
+  const size_t max_arena = mi_arenas_get_count(subproc);
+  if (max_arena == 0) return;
+
+  // allow only one thread to purge at a time (todo: allow concurrent purging?)
+  static mi_atomic_guard_t purge_guard;
+  mi_atomic_guard(&purge_guard)
+  {
+    // increase global expire: at most one purge per delay cycle
+    if (arenas_expire > now) { mi_atomic_storei64_release(&subproc->purge_expire, now + (delay/10)); }
+    const size_t arena_start = tseq % max_arena;
+    size_t max_purge_count = (visit_all ? max_arena : (max_arena/4)+1);
+    bool all_visited = true;
+    bool any_purged = false;
+    for (size_t _i = 0; _i < max_arena; _i++) {
+      size_t i = _i + arena_start;
+      if (i >= max_arena) { i -= max_arena; }
+      mi_arena_t* arena = mi_arena_from_index(subproc,i);
+      if (arena != NULL) {
+        const int purged = mi_arena_try_purge(arena, now, force);
+        if (purged >= 0) {      // purged, or arena expire is not yet reached
+          any_purged = true;
+          if (purged >= 1) {    // purged
+            if (max_purge_count <= 1) {
+              all_visited = false;
+              break;
+            }
+            max_purge_count--;
+          }
+        }
+      }
+    }
+    if (all_visited && !any_purged) {
+      mi_atomic_storei64_release(&subproc->purge_expire, (mi_msecs_t)0);
+    }
+  }
+}
+
+
+/* -----------------------------------------------------------
+  Visit all pages and blocks in a heap
+----------------------------------------------------------- */
+
+typedef struct mi_heap_visit_info_s {
+  mi_heap_t* heap;
+  mi_block_visit_fun* visitor;
+  void* arg;
+  bool visit_blocks;
+} mi_heap_visit_info_t;
+
+static bool mi_heap_visit_page(mi_page_t* page, mi_heap_visit_info_t* vinfo) {
+  mi_heap_area_t area;
+  _mi_heap_area_init(&area, page);
+  mi_assert_internal(vinfo->heap == mi_page_heap(page));
+  if (!vinfo->visitor(vinfo->heap, &area, NULL, area.block_size, vinfo->arg)) {
+    return false;
+  }
+  if (vinfo->visit_blocks) {
+    return _mi_theap_area_visit_blocks(&area, page, vinfo->visitor, vinfo->arg);
+  }
+  else {
+    return true;
+  }
+}
+
+static bool mi_heap_visit_page_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) {
+  MI_UNUSED(slice_count);
+  mi_heap_visit_info_t* vinfo = (mi_heap_visit_info_t*)arg;
+  mi_page_t* page = mi_arena_page_at_slice(arena, slice_index);
+  return mi_heap_visit_page(page, vinfo);
+}
+
+bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
+  mi_assert(visitor!=NULL);
+  if (visitor==NULL) return false;
+  if (heap==NULL) { heap = mi_heap_main(); }
+  // visit all pages in a heap
+  // we don't have to claim because we assume we are the only thread running (with this heap).
+  // (but we could atomically claim as well by first doing abandoned_reclaim and afterwards reabandoning).
+  mi_heap_visit_info_t visit_info = { heap, visitor, arg, visit_blocks };
+  bool ok = true;
+  mi_forall_arenas(heap, NULL, 0, arena) {
+    mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);
+    if (ok && arena_pages != NULL) {
+      if (abandoned_only) {
+        for (size_t bin = 0; ok && bin < MI_BIN_COUNT; bin++) {
+          // todo: if we had a single abandoned page map as well, this can be faster.
+          if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) {
+            ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info);
+          }
+        }
+      }
+      else {
+        ok = _mi_bitmap_forall_set(arena_pages->pages, &mi_heap_visit_page_at, arena, &visit_info);
+      }
+    }
+  }
+  mi_forall_arenas_end();
+  if (!ok) return false;
+
+  // visit abandoned pages in OS allocated memory
+  // (technically we don't need the initial lock as we assume we are the only thread running in this subproc)
+  mi_page_t* page = NULL;
+  mi_lock(&heap->os_abandoned_pages_lock) {
+    page = heap->os_abandoned_pages;
+  }
+  while (ok && page != NULL) {
+    mi_page_t* next = page->next;  // read upfront in case the visitor frees the page
+    ok = mi_heap_visit_page(page, &visit_info);
+    page = next;
+  }
+
+  return ok;
+}
+
+bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
+  return _mi_heap_visit_blocks(heap, false, visit_blocks, visitor, arg);
+}
+
+bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
+  return _mi_heap_visit_blocks(heap, true, visit_blocks, visitor, arg);
+}
+
+
+typedef struct mi_heap_delete_visit_info_s {
+  mi_heap_t*  heap_target;
+  mi_theap_t* theap_target;
+  mi_theap_t* theap;
+} mi_heap_delete_visit_info_t;
+
+static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) {
+  MI_UNUSED(block); MI_UNUSED(block_size); MI_UNUSED(heap);
+  mi_heap_delete_visit_info_t* info = (mi_heap_delete_visit_info_t*)arg;
+  mi_heap_t*  heap_target           = info->heap_target;
+  mi_theap_t* const theap           = NULL; // info->theap;       mi_assert_internal(_mi_theap_heap(theap) == heap);
+  mi_page_t*  const page            = (mi_page_t*)area->reserved1;
+
+  mi_page_claim_ownership(page);       // claim ownership
+  if (mi_page_is_abandoned(page)) {
+    _mi_arenas_page_unabandon(page,theap);
+  }
+  else {
+    page->next = page->prev = NULL;    // yikes.. better not to try to access this from a thread later on..
+    mi_page_set_theap(page,NULL);      // set threadid to abandoned
+  }
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(mi_page_is_owned(page));
+
+  if (page->used==0) {
+    // free the page
+    _mi_arenas_page_free(page, theap);
+  }
+  else if (heap_target==NULL) {
+    // destroy the page
+    page->used=0;                        // note: invariant `|local_free| + |free| == reserved - used`  does not hold in this case
+    _mi_arenas_page_free(page, theap);
+  }
+  else {
+    // move the page to `heap_target` as an abandoned page
+    // first remove it from the current heap
+    const size_t sbin = _mi_page_stats_bin(page);
+    size_t slice_index;
+    size_t slice_count;
+    mi_arena_pages_t* arena_pages = NULL;
+    mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
+    mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index));
+    mi_bitmap_clear(arena_pages->pages, slice_index);
+    if (theap != NULL) {
+      mi_theap_stat_decrease(theap, page_bins[sbin], 1);
+      mi_theap_stat_decrease(theap, pages, 1);
+    }
+    else {
+      mi_heap_stat_decrease((mi_heap_t*)heap, page_bins[_mi_page_stats_bin(page)], 1);
+      mi_heap_stat_decrease((mi_heap_t*)heap, pages, 1);
+    }
+    mi_theap_t* theap_target = info->theap_target;
+
+    // and then add it to the new target heap
+    mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);
+    if mi_unlikely(arena_pages_target==NULL) {
+      // if we cannot allocate this, we move it to the main heap instead (which does not require allocation)
+      heap_target = mi_heap_main();
+      theap_target = mi_heap_theap(heap_target);
+      arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);
+      mi_assert_internal(arena_pages_target!=NULL);
+    }
+    mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index));
+    mi_bitmap_set(arena_pages_target->pages, slice_index);
+    page->heap = heap_target;
+    mi_theap_stat_increase(theap_target, page_bins[sbin], 1);
+    mi_theap_stat_increase(theap_target, pages, 1);
+
+    // and abandon in the new heap
+    _mi_arenas_page_abandon(page,theap_target);
+  }
+  return true;
+}
+
+static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) {
+  mi_theap_t* const theap_target = (heap_target != NULL ? _mi_heap_theap(heap_target) : NULL);
+  // mi_theap_t* const theap = _mi_heap_theap(heap);
+  mi_heap_delete_visit_info_t info = { heap_target, theap_target, NULL };
+  _mi_heap_visit_blocks(heap, false, false, &mi_heap_delete_page, &info);
+  #if MI_DEBUG>1
+  // no more arena pages?
+  for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
+    mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
+    if (arena_pages!=NULL) {
+      mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages));
+    }
+  }
+  // nor os abandoned pages?
+  mi_lock(&heap->os_abandoned_pages_lock) {
+
+    mi_assert_internal(heap->os_abandoned_pages == NULL);
+  }
+  // nor arena abandoned pages?
+  for (size_t i = 0; i < MI_BIN_COUNT; i++) {
+    mi_assert_internal(mi_atomic_load_relaxed(&heap->abandoned_count[i])==0);
+  }
+  #endif
+}
+
+void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to) {
+  if (_mi_is_heap_main(heap_from)) return;
+  if (heap_to==NULL) { heap_to = mi_heap_main(); }
+  mi_heap_delete_pages(heap_from, heap_to);
+}
+
+void _mi_heap_destroy_pages(mi_heap_t* heap_from) {
+  if (_mi_is_heap_main(heap_from)) return;
+  mi_heap_delete_pages(heap_from, NULL);
+}
+
+/* -----------------------------------------------------------
+  Unloading and reloading an arena.
+----------------------------------------------------------- */
+/*
+static bool mi_arena_page_register(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) {
+  MI_UNUSED(arg); MI_UNUSED(slice_count);
+  mi_assert_internal(slice_count == 1);
+  mi_page_t* page = mi_arena_page_at_slice(arena, slice_index);
+  mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1));
+  if (!_mi_page_map_register(page)) return false; // break
+  mi_assert_internal(_mi_ptr_page(page)==page);
+  return true;
+}
+
+mi_decl_nodiscard static bool mi_arena_pages_reregister(mi_arena_t* arena) {
+  return _mi_bitmap_forall_set(arena->pages, &mi_arena_page_register, arena, NULL);
+}
+
+mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* full_size) {
+  mi_arena_t* arena = _mi_arena_from_id(arena_id);
+  if (arena==NULL) {
+    return false;
+  }
+  else if (!arena->is_exclusive) {
+    _mi_warning_message("cannot unload a non-exclusive arena (id %zu at %p)\n", arena_id, arena);
+    return false;
+  }
+  else if (arena->memid.memkind != MI_MEM_EXTERNAL) {
+    _mi_warning_message("can only unload managed arena's for external memory (id %zu at %p)\n", arena_id, arena);
+    return false;
+  }
+
+  // find accessed size
+  const size_t asize = mi_size_of_slices(mi_arena_used_slices(arena));
+  if (base != NULL) { *base = (void*)arena; }
+  if (full_size != NULL) { *full_size = arena->memid.mem.os.size;  }
+  if (accessed_size != NULL) { *accessed_size = asize; }
+
+  // adjust abandoned page count
+  mi_subproc_t* const subproc = arena->subproc;
+  for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
+    const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);
+    if (count > 0) { mi_atomic_decrement_acq_rel(&subproc->abandoned_count[bin]); }
+  }
+
+  // unregister the pages
+  _mi_page_map_unregister_range(arena, asize);
+
+  // set arena entry to NULL
+  const size_t count = mi_arenas_get_count(subproc);
+  for(size_t i = 0; i < count; i++) {
+    if (mi_arena_from_index(subproc, i) == arena) {
+      mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);
+      if (i + 1 == count) { // try adjust the count?
+        size_t expected = count;
+        mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, count-1);
+      }
+      break;
+    }
+  }
+  return true;
+}
+
+mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) {
+  // assume the memory area is already containing the arena
+  if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }
+  if (start == NULL || size == 0) return false;
+  mi_arena_t* arena = (mi_arena_t*)start;
+  mi_memid_t memid = arena->memid;
+  if (memid.memkind != MI_MEM_EXTERNAL) {
+    _mi_warning_message("can only reload arena's from external memory (%p)\n", arena);
+    return false;
+  }
+  if (memid.mem.os.base != start) {
+    _mi_warning_message("the reloaded arena base address differs from the external memory (arena: %p, external: %p)\n", arena, start);
+    return false;
+  }
+  if (memid.mem.os.size != size) {
+    _mi_warning_message("the reloaded arena size differs from the external memory (arena size: %zu, external size: %zu)\n", arena->memid.mem.os.size, size);
+    return false;
+  }
+  if (!arena->is_exclusive) {
+    _mi_warning_message("the reloaded arena is not exclusive\n");
+    return false;
+  }
+
+  // re-initialize
+  arena->is_exclusive = true;
+  arena->commit_fun = commit_fun;
+  arena->commit_fun_arg = commit_fun_arg;
+  arena->subproc = _mi_subproc();
+  if (!mi_arenas_add(arena->subproc, arena, arena_id)) {
+    return false;
+  }
+  if (!mi_arena_pages_reregister(arena)) {
+    // todo: clear arena entry in the subproc?
+    return false;
+  }
+
+  // adjust abandoned page count
+  for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
+    const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);
+    if (count > 0) { mi_atomic_decrement_acq_rel(&arena->subproc->abandoned_count[bin]); }
+  }
+
+  return true;
+}
+
+*/
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.c
new file mode 100644
index 00000000..67269e8b
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.c
@@ -0,0 +1,1949 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+/* ----------------------------------------------------------------------------
+Concurrent bitmap that can set/reset sequences of bits atomically
+---------------------------------------------------------------------------- */
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/bits.h"
+#include "mimalloc/prim.h"  // _mi_prim_thread_yield
+#include "bitmap.h"
+
+#ifndef MI_OPT_SIMD
+#define MI_OPT_SIMD   0
+#endif
+
+/* --------------------------------------------------------------------------------
+  bfields
+-------------------------------------------------------------------------------- */
+
+static inline size_t mi_bfield_ctz(mi_bfield_t x) {
+  return mi_ctz(x);
+}
+
+static inline size_t mi_bfield_clz(mi_bfield_t x) {
+  return mi_clz(x);
+}
+
+static inline size_t mi_bfield_popcount(mi_bfield_t x) {
+  return mi_popcount(x);
+}
+
+static inline mi_bfield_t mi_bfield_clear_least_bit(mi_bfield_t x) {
+  return (x & (x-1));
+}
+
+// find the least significant bit that is set (i.e. count trailing zero's)
+// return false if `x==0` (with `*idx` undefined) and true otherwise,
+// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
+static inline bool mi_bfield_find_least_bit(mi_bfield_t x, size_t* idx) {
+  return mi_bsf(x,idx);
+}
+
+// find the most significant bit that is set.
+// return false if `x==0` (with `*idx` undefined) and true otherwise,
+// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
+static inline bool mi_bfield_find_highest_bit(mi_bfield_t x, size_t* idx) {
+  return mi_bsr(x, idx);
+}
+
+
+
+// find each set bit in a bit field `x` and clear it, until it becomes zero.
+static inline bool mi_bfield_foreach_bit(mi_bfield_t* x, size_t* idx) {
+  const bool found = mi_bfield_find_least_bit(*x, idx);
+  *x = mi_bfield_clear_least_bit(*x);
+  return found;
+}
+
+static inline mi_bfield_t mi_bfield_zero(void) {
+  return 0;
+}
+
+static inline mi_bfield_t mi_bfield_one(void) {
+  return 1;
+}
+
+static inline mi_bfield_t mi_bfield_all_set(void) {
+  return ~((mi_bfield_t)0);
+}
+
+// mask of `bit_count` bits set shifted to the left by `shiftl`
+static inline mi_bfield_t mi_bfield_mask(size_t bit_count, size_t shiftl) {
+  mi_assert_internal(bit_count > 0);
+  mi_assert_internal(bit_count + shiftl <= MI_BFIELD_BITS);
+  mi_assert_internal(shiftl < MI_BFIELD_BITS);
+  const mi_bfield_t mask0 = (bit_count < MI_BFIELD_BITS ? (mi_bfield_one() << bit_count)-1 : mi_bfield_all_set());
+  return (mask0 << shiftl);
+}
+
+
+// ------- mi_bfield_atomic_set ---------------------------------------
+// the `_set` functions return also the count of bits that were already set (for commit statistics)
+// the `_clear` functions return also whether the new bfield is all clear or not (for the chunk_map)
+
+// Set a bit atomically. Returns `true` if the bit transitioned from 0 to 1
+static inline bool mi_bfield_atomic_set(_Atomic(mi_bfield_t)*b, size_t idx) {
+  mi_assert_internal(idx < MI_BFIELD_BITS);
+  const mi_bfield_t mask = mi_bfield_mask(1, idx);;
+  const mi_bfield_t old = mi_atomic_or_acq_rel(b, mask);
+  return ((old&mask) == 0);
+}
+
+// Clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0.
+// `all_clear` is set if the new bfield is zero.
+static inline bool mi_bfield_atomic_clear(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) {
+  mi_assert_internal(idx < MI_BFIELD_BITS);
+  const mi_bfield_t mask = mi_bfield_mask(1, idx);;
+  mi_bfield_t old = mi_atomic_and_acq_rel(b, ~mask);
+  if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }
+  return ((old&mask) == mask);
+}
+
+// Clear a bit but only when/once it is set. This is used by concurrent free's while
+// the page is abandoned and mapped. This can incure a busy wait :-( but it should
+// happen almost never (and is accounted for in the stats)
+static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_t idx) {
+  mi_assert_internal(idx < MI_BFIELD_BITS);
+  const mi_bfield_t mask = mi_bfield_mask(1, idx);;
+  mi_bfield_t old = mi_atomic_load_relaxed(b);
+  do {
+    if mi_unlikely((old&mask) == 0) {
+      old = mi_atomic_load_acquire(b);
+      if ((old&mask)==0) {
+        mi_subproc_stat_counter_increase(_mi_subproc(), pages_unabandon_busy_wait, 1);
+      }
+      while ((old&mask)==0) { // busy wait
+        _mi_prim_thread_yield(); 
+        old = mi_atomic_load_acquire(b);
+      }
+    }
+  } while (!mi_atomic_cas_weak_acq_rel(b,&old, (old&~mask)));
+  mi_assert_internal((old&mask)==mask);  // we should only clear when it was set
+}
+
+// Set a mask set of bits atomically, and return true of the mask bits transitioned from all 0's to 1's.
+// `already_set` contains the count of bits that were already set (used when committing ranges to account
+// statistics correctly).
+static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) {
+  mi_assert_internal(mask != 0);
+  mi_bfield_t old = mi_atomic_load_relaxed(b);
+  while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) {};  // try to atomically set the mask bits until success
+  if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); }
+  return ((old&mask) == 0);
+}
+
+// Clear a mask set of bits atomically, and return true of the mask bits transitioned from all 1's to 0's
+// `all_clear` is set to `true` if the new bfield became zero.
+static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, bool* all_clear) {
+  mi_assert_internal(mask != 0);
+  mi_bfield_t old = mi_atomic_load_relaxed(b);
+  while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) {};  // try to atomically clear the mask bits until success
+  if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }
+  return ((old&mask) == mask);
+}
+
+static inline bool mi_bfield_atomic_setX(_Atomic(mi_bfield_t)*b, size_t* already_set) {
+  const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_all_set());
+  if (already_set!=NULL) { *already_set = mi_bfield_popcount(old); }
+  return (old==0);
+}
+
+// static inline bool mi_bfield_atomic_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) {
+//   const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_zero());
+//   if (all_clear!=NULL) { *all_clear = true; }
+//   return (~old==0);
+// }
+
+// ------- mi_bfield_atomic_try_clear ---------------------------------------
+
+
+// Tries to clear a mask atomically, and returns true if the mask bits atomically transitioned from mask to 0
+// and false otherwise (leaving the bit field as is).
+// `all_clear` is set to `true` if the new bfield became zero.
+static inline bool mi_bfield_atomic_try_clear_mask_of(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t expect, bool* all_clear) {
+  mi_assert_internal(mask != 0);
+  // try to atomically clear the mask bits
+  do {
+    if ((expect & mask) != mask) {  // are all bits still set?
+      if (all_clear != NULL) { *all_clear = (expect == 0); }
+      return false;
+    }
+  } while (!mi_atomic_cas_weak_acq_rel(b, &expect, expect & ~mask));
+  if (all_clear != NULL) { *all_clear = ((expect & ~mask) == 0);  }
+  return true;
+}
+
+static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear) {
+  mi_assert_internal(mask != 0);
+  const mi_bfield_t expect = mi_atomic_load_relaxed(b);
+  return mi_bfield_atomic_try_clear_mask_of(b, mask, expect, all_clear);
+}
+
+// Tries to clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0
+// and `false` otherwise leaving the bfield `b` as-is.
+// `all_clear` is set to true if the new bfield became zero (and false otherwise)
+mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear(_Atomic(mi_bfield_t)* b, size_t idx, bool* all_clear) {
+  mi_assert_internal(idx < MI_BFIELD_BITS);
+  const mi_bfield_t mask = mi_bfield_one()<<idx;
+  return mi_bfield_atomic_try_clear_mask(b, mask, all_clear);
+}
+
+// Tries to clear a byte atomically, and returns true if the byte atomically transitioned from 0xFF to 0
+// `all_clear` is set to true if the new bfield became zero (and false otherwise)
+mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear8(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) {
+  mi_assert_internal(idx < MI_BFIELD_BITS);
+  mi_assert_internal((idx%8)==0);
+  const mi_bfield_t mask = ((mi_bfield_t)0xFF)<<idx;
+  return mi_bfield_atomic_try_clear_mask(b, mask, all_clear);
+}
+
+// Try to clear a full field of bits atomically, and return true all bits transitioned from all 1's to 0's.
+// and false otherwise leaving the bit field as-is.
+// `all_clear` is set to true if the new bfield became zero (which is always the case if successful).
+static inline bool mi_bfield_atomic_try_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) {
+  mi_bfield_t old = mi_bfield_all_set();
+  if (mi_atomic_cas_strong_acq_rel(b, &old, mi_bfield_zero())) {
+    if (all_clear != NULL) { *all_clear = true; }
+    return true;
+  }
+  else return false;
+}
+
+
+// ------- mi_bfield_atomic_is_set ---------------------------------------
+
+// Check if a bit is set
+static inline bool mi_bfield_atomic_is_set(const _Atomic(mi_bfield_t)*b, const size_t idx) {
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
+  return ((x & mi_bfield_mask(1,idx)) != 0);
+}
+
+// Check if a bit is clear
+static inline bool mi_bfield_atomic_is_clear(const _Atomic(mi_bfield_t)*b, const size_t idx) {
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
+  return ((x & mi_bfield_mask(1, idx)) == 0);
+}
+
+// Check if a bit is xset
+static inline bool mi_bfield_atomic_is_xset(mi_xset_t set, const _Atomic(mi_bfield_t)*b, const size_t idx) {
+  if (set) return mi_bfield_atomic_is_set(b, idx);
+      else return mi_bfield_atomic_is_clear(b, idx);
+}
+
+// Check if all bits corresponding to a mask are set.
+static inline bool mi_bfield_atomic_is_set_mask(const _Atomic(mi_bfield_t)* b, mi_bfield_t mask) {
+  mi_assert_internal(mask != 0);
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
+  return ((x & mask) == mask);
+}
+
+// Check if all bits corresponding to a mask are clear.
+static inline bool mi_bfield_atomic_is_clear_mask(const _Atomic(mi_bfield_t)* b, mi_bfield_t mask) {
+  mi_assert_internal(mask != 0);
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
+  return ((x & mask) == 0);
+}
+
+// Check if all bits corresponding to a mask are set/cleared.
+static inline bool mi_bfield_atomic_is_xset_mask(mi_xset_t set, const _Atomic(mi_bfield_t)* b, mi_bfield_t mask) {
+  mi_assert_internal(mask != 0);
+  if (set) return mi_bfield_atomic_is_set_mask(b, mask);
+      else return mi_bfield_atomic_is_clear_mask(b, mask);
+}
+
+// Count bits in a mask
+static inline size_t mi_bfield_atomic_popcount_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask) {
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
+  return mi_bfield_popcount(x & mask);
+}
+
+
+/* --------------------------------------------------------------------------------
+ bitmap chunks
+-------------------------------------------------------------------------------- */
+
+// ------- mi_bchunk_set ---------------------------------------
+
+// Set a single bit
+static inline bool mi_bchunk_set(mi_bchunk_t* chunk, size_t cidx, size_t* already_set) {
+  mi_assert_internal(cidx < MI_BCHUNK_BITS);
+  const size_t i = cidx / MI_BFIELD_BITS;
+  const size_t idx = cidx % MI_BFIELD_BITS;
+  const bool was_clear = mi_bfield_atomic_set(&chunk->bfields[i], idx);
+  if (already_set != NULL) { *already_set = (was_clear ? 0 : 1); }
+  return was_clear;
+}
+
+// Set `0 < n <= MI_BFIELD_BITS`, and return true of the mask bits transitioned from all 0's to 1's.
+// `already_set` contains the count of bits that were already set (used when committing ranges to account
+// statistics correctly).
+// Can cross over two bfields.
+static inline bool mi_bchunk_setNX(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) {
+  mi_assert_internal(cidx < MI_BCHUNK_BITS);
+  mi_assert_internal(n > 0 && n <= MI_BFIELD_BITS);
+  const size_t i = cidx / MI_BFIELD_BITS;
+  const size_t idx = cidx % MI_BFIELD_BITS;
+  if mi_likely(idx + n <= MI_BFIELD_BITS) {
+    // within one field
+    return mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(n,idx), already_set);
+  }
+  else {
+    // spanning two fields
+    const size_t m = MI_BFIELD_BITS - idx;  // bits to clear in the first field
+    mi_assert_internal(m < n);
+    mi_assert_internal(i < MI_BCHUNK_FIELDS - 1);
+    mi_assert_internal(idx + m <= MI_BFIELD_BITS);
+    size_t already_set1;
+    const bool all_set1 = mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &already_set1);
+    mi_assert_internal(n - m > 0);
+    mi_assert_internal(n - m < MI_BFIELD_BITS);
+    size_t already_set2;
+    const bool all_set2 = mi_bfield_atomic_set_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &already_set2);
+    if (already_set != NULL) { *already_set = already_set1 + already_set2; }
+    return (all_set1 && all_set2);
+  }
+}
+
+// Set a sequence of `n` bits within a chunk.
+// Returns true if all bits transitioned from 0 to 1 (or 1 to 0).
+mi_decl_noinline static bool mi_bchunk_xsetNC(mi_xset_t set, mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* palready_set, bool* pmaybe_all_clear) {
+  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
+  mi_assert_internal(n>0);
+  bool all_transition = true;
+  bool maybe_all_clear = true;
+  size_t total_already_set = 0;
+  size_t idx   = cidx % MI_BFIELD_BITS;
+  size_t field = cidx / MI_BFIELD_BITS;
+  while (n > 0) {
+    size_t m = MI_BFIELD_BITS - idx;   // m is the bits to xset in this field
+    if (m > n) { m = n; }
+    mi_assert_internal(idx + m <= MI_BFIELD_BITS);
+    mi_assert_internal(field < MI_BCHUNK_FIELDS);
+    const mi_bfield_t mask = mi_bfield_mask(m, idx);
+    size_t already_set = 0;
+    bool all_clear = false;
+    const bool transition = (set ? mi_bfield_atomic_set_mask(&chunk->bfields[field], mask, &already_set)
+                                 : mi_bfield_atomic_clear_mask(&chunk->bfields[field], mask, &all_clear));
+    mi_assert_internal((transition && already_set == 0) || (!transition && already_set > 0));
+    all_transition = all_transition && transition;
+    total_already_set += already_set;
+    maybe_all_clear = maybe_all_clear && all_clear;
+    // next field
+    field++;
+    idx = 0;
+    mi_assert_internal(m <= n);
+    n -= m;
+  }
+  if (palready_set!=NULL) { *palready_set = total_already_set; }
+  if (pmaybe_all_clear!=NULL) { *pmaybe_all_clear = maybe_all_clear; }
+  return all_transition;
+}
+
+static inline bool mi_bchunk_setN(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) {
+  mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS);
+  if (n==1) return mi_bchunk_set(chunk, cidx, already_set);
+  // if (n==8 && (cidx%8) == 0) return mi_bchunk_set8(chunk, cidx, already_set);
+  // if (n==MI_BFIELD_BITS) return mi_bchunk_setX(chunk, cidx, already_set);
+  if (n<=MI_BFIELD_BITS) return mi_bchunk_setNX(chunk, cidx, n, already_set);
+  return mi_bchunk_xsetNC(MI_BIT_SET, chunk, cidx, n, already_set, NULL);
+}
+
+// ------- mi_bchunk_clear ---------------------------------------
+
+static inline bool mi_bchunk_clear(mi_bchunk_t* chunk, size_t cidx, bool* all_clear) {
+  mi_assert_internal(cidx < MI_BCHUNK_BITS);
+  const size_t i = cidx / MI_BFIELD_BITS;
+  const size_t idx = cidx % MI_BFIELD_BITS;
+  return mi_bfield_atomic_clear(&chunk->bfields[i], idx, all_clear);
+}
+
+static inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) {
+  mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS);
+  if (n==1) return mi_bchunk_clear(chunk, cidx, maybe_all_clear);
+  // if (n==8) return mi_bchunk_clear8(chunk, cidx, maybe_all_clear);
+  // if (n==MI_BFIELD_BITS) return mi_bchunk_clearX(chunk, cidx, maybe_all_clear);
+  // TODO: implement mi_bchunk_xsetNX instead of setNX
+  return mi_bchunk_xsetNC(MI_BIT_CLEAR, chunk, cidx, n, NULL, maybe_all_clear);
+}
+
+// Check if a sequence of `n` bits within a chunk are all set/cleared.
+// This can cross bfield's
+mi_decl_noinline static size_t mi_bchunk_popcountNC(mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) {
+  mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS);
+  size_t count = 0;
+  while (n > 0) {
+    size_t m = MI_BFIELD_BITS - idx;   // m is the bits to xset in this field
+    if (m > n) { m = n; }
+    mi_assert_internal(idx + m <= MI_BFIELD_BITS);
+    mi_assert_internal(field_idx < MI_BCHUNK_FIELDS);
+    const size_t mask = mi_bfield_mask(m, idx);
+    count += mi_bfield_atomic_popcount_mask(&chunk->bfields[field_idx], mask);
+    // next field
+    field_idx++;
+    idx = 0;
+    n -= m;
+  }
+  return count;
+}
+
+// Count set bits a sequence of `n` bits.
+static inline size_t mi_bchunk_popcountN(mi_bchunk_t* chunk, size_t cidx, size_t n) {
+  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
+  mi_assert_internal(n>0);
+  if (n==0) return 0;
+  const size_t i = cidx / MI_BFIELD_BITS;
+  const size_t idx = cidx % MI_BFIELD_BITS;
+  if (n==1) { return (mi_bfield_atomic_is_set(&chunk->bfields[i], idx) ? 1 : 0); }
+  if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_popcount_mask(&chunk->bfields[i], mi_bfield_mask(n, idx)); }
+  return mi_bchunk_popcountNC(chunk, i, idx, n);
+}
+
+
+// ------- mi_bchunk_is_xset ---------------------------------------
+
+// Check if a sequence of `n` bits within a chunk are all set/cleared.
+// This can cross bfield's
+mi_decl_noinline static bool mi_bchunk_is_xsetNC(mi_xset_t set, const mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) {
+  mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS);
+  while (n > 0) {
+    size_t m = MI_BFIELD_BITS - idx;   // m is the bits to xset in this field
+    if (m > n) { m = n; }
+    mi_assert_internal(idx + m <= MI_BFIELD_BITS);
+    mi_assert_internal(field_idx < MI_BCHUNK_FIELDS);
+    const size_t mask = mi_bfield_mask(m, idx);
+    if (!mi_bfield_atomic_is_xset_mask(set, &chunk->bfields[field_idx], mask)) {
+      return false;
+    }
+    // next field
+    field_idx++;
+    idx = 0;
+    n -= m;
+  }
+  return true;
+}
+
+// Check if a sequence of `n` bits within a chunk are all set/cleared.
+static inline bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, size_t cidx, size_t n) {
+  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
+  mi_assert_internal(n>0);
+  if (n==0) return true;
+  const size_t i = cidx / MI_BFIELD_BITS;
+  const size_t idx = cidx % MI_BFIELD_BITS;
+  if (n==1) { return mi_bfield_atomic_is_xset(set, &chunk->bfields[i], idx); }
+  if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_is_xset_mask(set, &chunk->bfields[i], mi_bfield_mask(n, idx)); }
+  return mi_bchunk_is_xsetNC(set, chunk, i, idx, n);
+}
+
+
+// ------- mi_bchunk_try_clear  ---------------------------------------
+
+// Clear `0 < n <= MI_BITFIELD_BITS`. Can cross over a bfield boundary.
+static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {
+  mi_assert_internal(cidx < MI_BCHUNK_BITS);
+  mi_assert_internal(n <= MI_BFIELD_BITS);
+  const size_t i = cidx / MI_BFIELD_BITS;
+  const size_t idx = cidx % MI_BFIELD_BITS;
+  if mi_likely(idx + n <= MI_BFIELD_BITS) {
+    // within one field
+    return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear);
+  }
+  else {
+    // spanning two fields (todo: use double-word atomic ops?)
+    const size_t m = MI_BFIELD_BITS - idx;  // bits to clear in the first field
+    mi_assert_internal(m < n);
+    mi_assert_internal(i < MI_BCHUNK_FIELDS - 1);
+    bool field1_is_clear;
+    if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear)) return false;
+    // try the second field as well
+    mi_assert_internal(n - m > 0);
+    mi_assert_internal(n - m < MI_BFIELD_BITS);
+    bool field2_is_clear;
+    if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear)) {
+      // we failed to clear the second field, restore the first one
+      mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), NULL);
+      return false;
+    }
+    if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = field1_is_clear && field2_is_clear;  }
+    return true;
+  }
+}
+
+// Clear a full aligned bfield.
+// static inline bool mi_bchunk_try_clearX(mi_bchunk_t* chunk, size_t cidx, bool* pmaybe_all_clear) {
+//   mi_assert_internal(cidx < MI_BCHUNK_BITS);
+//   mi_assert_internal((cidx%MI_BFIELD_BITS) == 0);
+//   const size_t i = cidx / MI_BFIELD_BITS;
+//   return mi_bfield_atomic_try_clearX(&chunk->bfields[i], pmaybe_all_clear);
+// }
+
+// Try to atomically clear a sequence of `n` bits within a chunk.
+// Returns true if all bits transitioned from 1 to 0,
+// and false otherwise leaving all bit fields as is.
+// Note: this is the complex one as we need to unwind partial atomic operations if we fail halfway..
+// `maybe_all_clear` is set to `true` if all the bfields involved become zero.
+mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {
+  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
+  mi_assert_internal(n>0);
+  if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = true; }
+  if (n==0) return true;
+
+  // first field
+  const size_t start_idx = cidx % MI_BFIELD_BITS;
+  const size_t start_field = cidx / MI_BFIELD_BITS;
+  size_t field = start_field;
+  size_t m = MI_BFIELD_BITS - start_idx;   // m are the bits to clear in this field
+  if (m > n) { m = n; }
+  mi_assert_internal(start_idx + m <= MI_BFIELD_BITS);
+  mi_assert_internal(start_field < MI_BCHUNK_FIELDS);
+  const mi_bfield_t mask_start = mi_bfield_mask(m, start_idx);
+  bool maybe_all_clear;
+  if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear)) return false;
+
+  // done?
+  mi_assert_internal(m <= n);
+  n -= m;
+
+  // continue with mid fields and last field: if these fail we need to recover by unsetting previous fields
+  // mid fields?
+  while (n >= MI_BFIELD_BITS) {
+    field++;
+    mi_assert_internal(field < MI_BCHUNK_FIELDS);
+    bool field_is_clear;
+    if (!mi_bfield_atomic_try_clearX(&chunk->bfields[field], &field_is_clear)) goto restore;
+    maybe_all_clear = maybe_all_clear && field_is_clear;
+    n -= MI_BFIELD_BITS;
+  }
+
+  // last field?
+  if (n > 0) {
+    mi_assert_internal(n < MI_BFIELD_BITS);
+    field++;
+    mi_assert_internal(field < MI_BCHUNK_FIELDS);
+    const mi_bfield_t mask_end = mi_bfield_mask(n, 0);
+    bool field_is_clear;
+    if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear)) goto restore;
+    maybe_all_clear = maybe_all_clear && field_is_clear;
+  }
+
+  if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = maybe_all_clear; }
+  return true;
+
+restore:
+  // `field` is the index of the field that failed to set atomically; we need to restore all previous fields
+  mi_assert_internal(field > start_field);
+  while( field > start_field) {
+    field--;
+    if (field == start_field) {
+      mi_bfield_atomic_set_mask(&chunk->bfields[field], mask_start, NULL);
+    }
+    else {
+      mi_bfield_atomic_setX(&chunk->bfields[field], NULL);  // mid-field: set all bits again
+    }
+  }
+  return false;
+}
+
+
+static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) {
+  mi_assert_internal(n>0);
+  // if (n==MI_BFIELD_BITS) return mi_bchunk_try_clearX(chunk, cidx, maybe_all_clear);
+  if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear);
+  return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear);
+}
+
+
+// ------- mi_bchunk_try_find_and_clear ---------------------------------------
+
+#if MI_OPT_SIMD && defined(__AVX2__)
+mi_decl_maybe_unused static inline __m256i mi_mm256_zero(void) {
+  return _mm256_setzero_si256();
+}
+mi_decl_maybe_unused static inline __m256i mi_mm256_ones(void) {
+  return _mm256_set1_epi64x(~0);
+}
+mi_decl_maybe_unused static inline bool mi_mm256_is_ones(__m256i vec) {
+  return _mm256_testc_si256(vec, _mm256_cmpeq_epi32(vec, vec));
+}
+mi_decl_maybe_unused static inline bool mi_mm256_is_zero( __m256i vec) {
+  return _mm256_testz_si256(vec,vec);
+}
+#endif
+
+static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {
+  mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);
+  // note: this must be acquire (and not relaxed), or otherwise the AVX code below can loop forever
+  // as the compiler won't reload the registers vec1 and vec2 from memory again.
+  const mi_bfield_t b = mi_atomic_load_acquire(&chunk->bfields[chunk_idx]);
+  size_t idx;
+  if (mi_bfield_find_least_bit(b, &idx)) {           // find the least bit
+    if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], mi_bfield_mask(1,idx), b, NULL)) {  // clear it atomically
+      *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;
+      mi_assert_internal(*pidx < MI_BCHUNK_BITS);
+      return true;
+    }
+  }
+  return false;
+}
+
+// Find least 1-bit in a chunk and try to clear it atomically
+// set `*pidx` to the bit index (0 <= *pidx < MI_BCHUNK_BITS) on success.
+// This is used to find free slices and abandoned pages and should be efficient.
+// todo: try neon version
+static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx) {
+  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
+  for(int tries=0; tries<4; tries++) {   // paranoia: at most 4 tries
+    const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
+    const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF  : 0)
+    const uint32_t mask = ~_mm256_movemask_epi8(vcmp);  // mask of most significant bit of each byte (so each 8 bits are all set or clear)
+    // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)
+    if (mask==0) return false;
+    mi_assert_internal((_tzcnt_u32(mask)%8) == 0); // tzcnt == 0, 8, 16, or 24
+    const size_t chunk_idx = _tzcnt_u32(mask) / 8;
+    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;
+    // try again
+    // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded
+    // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)
+    #if defined(__GNUC__)
+    __asm __volatile ("" : : "g"(chunk) : "memory");
+    #endif
+  }
+  #elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
+  for(int tries=0; tries<4; tries++) {   // paranoia: at most 4 tries
+    size_t chunk_idx = 0;
+    #if 0
+    // one vector at a time
+    __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
+    if (mi_mm256_is_zero(vec)) {
+      chunk_idx += 4;
+      vec = _mm256_load_si256(((const __m256i*)chunk->bfields) + 1);
+    }
+    const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF  : 0)
+    const uint32_t mask = ~_mm256_movemask_epi8(vcmp);  // mask of most significant bit of each byte (so each 8 bits are all set or clear)
+    // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)
+    if (mask==0) return false;
+    mi_assert_internal((_tzcnt_u32(mask)%8) == 0); // tzcnt == 0, 8, 16, or 24
+    chunk_idx += _tzcnt_u32(mask) / 8;
+    #else
+    // a cache line is 64b so we can just as well load all at the same time
+    const __m256i vec1  = _mm256_load_si256((const __m256i*)chunk->bfields);
+    const __m256i vec2  = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);
+    const __m256i cmpv  = mi_mm256_zero();
+    const __m256i vcmp1 = _mm256_cmpeq_epi64(vec1, cmpv); // (elem64 == 0 ? 0xFF  : 0)
+    const __m256i vcmp2 = _mm256_cmpeq_epi64(vec2, cmpv); // (elem64 == 0 ? 0xFF  : 0)
+    const uint32_t mask1 = ~_mm256_movemask_epi8(vcmp1);  // mask of most significant bit of each byte (so each 8 bits are all set or clear)
+    const uint32_t mask2 = ~_mm256_movemask_epi8(vcmp2);  // mask of most significant bit of each byte (so each 8 bits are all set or clear)
+    const uint64_t mask = ((uint64_t)mask2 << 32) | mask1;
+    // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)
+    if (mask==0) return false;
+    mi_assert_internal((_tzcnt_u64(mask)%8) == 0); // tzcnt == 0, 8, 16, 24 , ..
+    chunk_idx = mi_ctz(mask) / 8;
+    #endif
+    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;
+    // try again
+    // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded
+    // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)
+    #if defined(__GNUC__)
+    __asm __volatile ("" : : "g"(chunk) : "memory");
+    #endif
+  }
+  #elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64
+  for(int tries=0; tries<4; tries++) {   // paranoia: at most 4 tries
+    // a cache line is 64b so we can just as well load all at the same time (?)
+    const uint64x2_t vzero1_lo = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields));        // 2x64 bit is_zero
+    const uint64x2_t vzero1_hi = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 2));    // 2x64 bit is_zero
+    const uint64x2_t vzero2_lo = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 4));    // 2x64 bit is_zero
+    const uint64x2_t vzero2_hi = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 6));    // 2x64 bit is_zero
+    const uint32x4_t vzero1    = vuzp1q_u32(vreinterpretq_u32_u64(vzero1_lo),vreinterpretq_u32_u64(vzero1_hi)); // unzip even elements: narrow to 4x32 bit is_zero ()
+    const uint32x4_t vzero2    = vuzp1q_u32(vreinterpretq_u32_u64(vzero2_lo),vreinterpretq_u32_u64(vzero2_hi)); // unzip even elements: narrow to 4x32 bit is_zero ()
+    const uint32x4_t vzero1x   = vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(vzero1), 24));        // shift-right 2x32bit elem by 24: lo 16 bits contain the 2 lo bytes
+    const uint32x4_t vzero2x   = vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(vzero2), 24));
+    const uint16x8_t vzero12   = vreinterpretq_u16_u32(vuzp1q_u32(vzero1x,vzero2x));                           // unzip even 32-bit elements into one vector
+    const uint8x8_t  vzero     = vmovn_u16(vzero12);                                                           // narrow the bottom 16-bits
+    const uint64_t mask = ~vget_lane_u64(vreinterpret_u64_u8(vzero), 0);  // 1 byte for each bfield (0xFF => bfield has a bit set)
+    if (mask==0) return false;
+    mi_assert_internal((mi_ctz(mask)%8) == 0); // tzcnt == 0, 8, 16, 24 , ..
+    const size_t chunk_idx = mi_ctz(mask) / 8;
+    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;
+    // try again
+    // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded
+    // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)
+    #if defined(__GNUC__)
+    __asm __volatile ("" : : "g"(chunk) : "memory");
+    #endif
+  }
+  #else
+  for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
+    if (mi_bchunk_try_find_and_clear_at(chunk, i, pidx)) return true;
+  }
+  #endif
+  return false;  
+}
+
+static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
+  mi_assert_internal(n==1); MI_UNUSED(n);
+  return mi_bchunk_try_find_and_clear(chunk, pidx);
+}
+
+mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {
+  const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[chunk_idx]);
+  // has_set8 has low bit in each byte set if the byte in x == 0xFF
+  const mi_bfield_t has_set8 =
+    ((~b - MI_BFIELD_LO_BIT8) &      // high bit set if byte in x is 0xFF or < 0x7F
+     (b  & MI_BFIELD_HI_BIT8))       // high bit set if byte in x is >= 0x80
+     >> 7;                           // shift high bit to low bit
+  size_t idx;
+  if (mi_bfield_find_least_bit(has_set8, &idx)) { // find least 1-bit
+    mi_assert_internal(idx <= (MI_BFIELD_BITS - 8));
+    mi_assert_internal((idx%8)==0);
+    if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], (mi_bfield_t)0xFF << idx, b, NULL)) {  // unset the byte atomically
+      *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;
+      mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);
+      return true;
+    }
+  }
+  return false;
+}
+
+// find least aligned byte in a chunk with all bits set, and try unset it atomically
+// set `*pidx` to its bit index (0 <= *pidx < MI_BCHUNK_BITS) on success.
+// Used to find medium size pages in the free blocks.
+// todo: try neon version
+static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx) {
+  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
+  while (true) {
+    // since a cache-line is 64b, load all at once
+    const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);
+    const __m256i vec2 = _mm256_load_si256((const __m256i*)chunk->bfields+1);
+    const __m256i cmpv = mi_mm256_ones();
+    const __m256i vcmp1 = _mm256_cmpeq_epi8(vec1, cmpv); // (byte == ~0 ? 0xFF : 0)
+    const __m256i vcmp2 = _mm256_cmpeq_epi8(vec2, cmpv); // (byte == ~0 ? 0xFF : 0)
+    const uint32_t mask1 = _mm256_movemask_epi8(vcmp1);    // mask of most significant bit of each byte
+    const uint32_t mask2 = _mm256_movemask_epi8(vcmp2);    // mask of most significant bit of each byte
+    const uint64_t mask = ((uint64_t)mask2 << 32) | mask1;
+    // mask is inverted, so each bit is 0xFF iff the corresponding byte has a bit set (and thus can be cleared)
+    if (mask==0) return false;
+    const size_t bidx = _tzcnt_u64(mask);          // byte-idx of the byte in the chunk
+    const size_t chunk_idx = bidx / 8;
+    const size_t idx = (bidx % 8)*8;
+    mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);
+    if mi_likely(mi_bfield_atomic_try_clear8(&chunk->bfields[chunk_idx], idx, NULL)) {  // clear it atomically
+      *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;
+      mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);
+      return true;
+    }
+    // try again
+    // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded  }
+  }
+  #else
+    for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
+      if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx)) return true;
+    }
+    return false;
+  #endif
+}
+
+static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
+  mi_assert_internal(n==8); MI_UNUSED(n);
+  return mi_bchunk_try_find_and_clear8(chunk, pidx);
+}
+
+
+// find a sequence of `n` bits in a chunk with `0 < n <= MI_BFIELD_BITS` with all bits set,
+// and try to clear them atomically.
+// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.
+// will cross bfield boundaries.
+mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
+  if (n == 0 || n > MI_BFIELD_BITS) return false;
+  const mi_bfield_t mask = mi_bfield_mask(n, 0);
+  // for all fields in the chunk
+  for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
+    mi_bfield_t b0 = mi_atomic_load_relaxed(&chunk->bfields[i]);
+    mi_bfield_t b = b0;
+    size_t idx;
+
+    // is there a range inside the field?
+    while (mi_bfield_find_least_bit(b, &idx)) { // find least 1-bit
+      if (idx + n > MI_BFIELD_BITS) break; // too short: maybe cross over, or continue with the next field
+
+      const size_t bmask = mask<<idx;
+      mi_assert_internal(bmask>>idx == mask);
+      if ((b&bmask) == bmask) { // found a match with all bits set, try clearing atomically
+        if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[i], bmask, b0, NULL)) {
+          *pidx = (i*MI_BFIELD_BITS) + idx;
+          mi_assert_internal(*pidx < MI_BCHUNK_BITS);
+          mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);
+          return true;
+        }
+        else {
+          // if we failed to atomically commit, reload b and try again from the start
+          b = b0 = mi_atomic_load_acquire(&chunk->bfields[i]);
+        }
+      }
+      else {
+        // advance by clearing the least run of ones, for example, with n>=4, idx=2:
+        // b             = 1111 1101 1010 1100
+        // .. + (1<<idx) = 1111 1101 1011 0000
+        // .. & b        = 1111 1101 1010 0000
+        b = b & (b + (mi_bfield_one() << idx));
+      }
+    }
+
+    // check if we can cross into the next bfield
+    if (b!=0 && i < MI_BCHUNK_FIELDS-1) {
+      const size_t post = mi_bfield_clz(~b);
+      if (post > 0) {
+        const size_t pre = mi_bfield_ctz(~mi_atomic_load_relaxed(&chunk->bfields[i+1]));
+        if (post + pre >= n) {
+          // it fits -- try to claim it atomically
+          const size_t cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - post);
+          if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL)) {
+            // we cleared all atomically
+            *pidx = cidx;
+            mi_assert_internal(*pidx < MI_BCHUNK_BITS);
+            mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);
+            return true;
+          }
+        }
+      }
+    }
+  }
+  return false;
+}
+
+// find a sequence of `n` bits in a chunk with `n <= MI_BCHUNK_BITS` with all bits set,
+// and try to clear them atomically.
+// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.
+// This can cross bfield boundaries.
+static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
+  if (n == 0 || n > MI_BCHUNK_BITS) return false;  // cannot be more than a chunk
+
+  // we first scan ahead to see if there is a range of `n` set bits, and only then try to clear atomically
+  mi_assert_internal(n>0);
+  const size_t skip_count = (n-1)/MI_BFIELD_BITS;
+  size_t cidx;
+  for (size_t i = 0; i < MI_BCHUNK_FIELDS - skip_count; i++)
+  {
+    size_t m = n;   // bits to go
+
+    // first field
+    mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
+    size_t ones = mi_bfield_clz(~b);
+
+    cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - ones);  // start index
+    if (ones >= m) {
+      // we found enough bits already!
+      m = 0;
+    }
+    else if (ones > 0) {
+      // keep scanning further fields until we have enough bits
+      m -= ones;
+      size_t j = 1;   // field count from i
+      while (i+j < MI_BCHUNK_FIELDS) {
+        mi_assert_internal(m > 0);
+        b = mi_atomic_load_relaxed(&chunk->bfields[i+j]);
+        ones = mi_bfield_ctz(~b);
+        if (ones >= m) {
+          // we found enough bits
+          m = 0;
+          break;
+        }
+        else if (ones == MI_BFIELD_BITS) {
+          // not enough yet, proceed to the next field
+          j++;
+          m -= MI_BFIELD_BITS;
+        }
+        else {
+          // the range was not enough, start from scratch
+          i = i + j - 1;  // no need to re-scan previous fields, except the last one (with clz this time)
+          mi_assert_internal(m>0);
+          break;
+        }
+      }
+    }
+
+    // did we find a range?
+    if (m==0) {
+      if (mi_bchunk_try_clearN(chunk, cidx, n, NULL)) {
+        // we cleared all atomically
+        *pidx = cidx;
+        mi_assert_internal(*pidx < MI_BCHUNK_BITS);
+        mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);
+        return true;
+      }
+      // note: if we fail for a small `n` on the first field, we don't rescan that field (as `i` is incremented)
+    }
+    // otherwise continue searching
+  }
+  return false;
+}
+
+
+
+// ------- mi_bchunk_clear_once_set ---------------------------------------
+
+static inline void mi_bchunk_clear_once_set(mi_bchunk_t* chunk, size_t cidx) {
+  mi_assert_internal(cidx < MI_BCHUNK_BITS);
+  const size_t i = cidx / MI_BFIELD_BITS;
+  const size_t idx = cidx % MI_BFIELD_BITS;
+  mi_bfield_atomic_clear_once_set(&chunk->bfields[i], idx);
+}
+
+
+// ------- mi_bitmap_all_are_clear ---------------------------------------
+
+
+// are all bits in a bitmap chunk clear?
+static inline bool mi_bchunk_all_are_clear_relaxed(mi_bchunk_t* chunk) {
+  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
+  const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
+  return mi_mm256_is_zero(vec);
+  #elif MI_OPT_SIMD &&  defined(__AVX2__) && (MI_BCHUNK_BITS==512)
+  // a 64b cache-line contains the entire chunk anyway so load both at once
+  const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);
+  const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);
+  return (mi_mm256_is_zero(_mm256_or_si256(vec1,vec2)));
+  #elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64
+  const uint64x2_t v0 = vld1q_u64((uint64_t*)chunk->bfields);
+  const uint64x2_t v1 = vld1q_u64((uint64_t*)chunk->bfields + 2);
+  const uint64x2_t v2 = vld1q_u64((uint64_t*)chunk->bfields + 4);
+  const uint64x2_t v3 = vld1q_u64((uint64_t*)chunk->bfields + 6);
+  const uint64x2_t v  = vorrq_u64(vorrq_u64(v0,v1),vorrq_u64(v2,v3));
+  return (vmaxvq_u32(vreinterpretq_u32_u64(v)) == 0);
+  #else
+  for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
+    if (mi_atomic_load_relaxed(&chunk->bfields[i]) != 0) return false;
+  }
+  return true;
+  #endif
+}
+
+// are all bits in a bitmap chunk set?
+static inline bool mi_bchunk_all_are_set_relaxed(mi_bchunk_t* chunk) {
+#if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
+  const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
+  return mi_mm256_is_ones(vec);
+#elif MI_OPT_SIMD &&  defined(__AVX2__) && (MI_BCHUNK_BITS==512)
+  // a 64b cache-line contains the entire chunk anyway so load both at once
+  const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);
+  const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);
+  return (mi_mm256_is_ones(_mm256_and_si256(vec1, vec2)));
+#elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64
+  const uint64x2_t v0 = vld1q_u64((uint64_t*)chunk->bfields);
+  const uint64x2_t v1 = vld1q_u64((uint64_t*)chunk->bfields + 2);
+  const uint64x2_t v2 = vld1q_u64((uint64_t*)chunk->bfields + 4);
+  const uint64x2_t v3 = vld1q_u64((uint64_t*)chunk->bfields + 6);
+  const uint64x2_t v  = vandq_u64(vandq_u64(v0,v1),vandq_u64(v2,v3));
+  return (vminvq_u32(vreinterpretq_u32_u64(v)) == 0xFFFFFFFFUL);
+#else
+  for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
+    if (~mi_atomic_load_relaxed(&chunk->bfields[i]) != 0) return false;
+  }
+  return true;
+#endif
+}
+
+
+static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) {
+  for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) {
+    i--;
+    mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
+    size_t idx;
+    if (mi_bsr(b, &idx)) {
+      *pidx = (i*MI_BFIELD_BITS) + idx;
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) {
+  for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) {
+    i--;
+    mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
+    size_t idx;
+    if (mi_bsr(~b, &idx)) {
+      *pidx = (i*MI_BFIELD_BITS) + idx;
+      return true;
+    }
+  }
+  return false;
+}
+
+static size_t mi_bchunk_popcount(mi_bchunk_t* chunk) {
+  size_t popcount = 0;
+  for (size_t i = 0; i < MI_BCHUNK_FIELDS; i++) {
+    const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
+    popcount += mi_bfield_popcount(b);
+  }
+  return popcount;
+}
+
+
+/* --------------------------------------------------------------------------------
+ bitmap chunkmap
+-------------------------------------------------------------------------------- */
+
+static void mi_bitmap_chunkmap_set(mi_bitmap_t* bitmap, size_t chunk_idx) {
+  mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap));
+  mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL);
+}
+
+static bool mi_bitmap_chunkmap_try_clear(mi_bitmap_t* bitmap, size_t chunk_idx) {
+  mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap));
+  // check if the corresponding chunk is all clear
+  if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) return false;
+  // clear the chunkmap bit
+  mi_bchunk_clear(&bitmap->chunkmap, chunk_idx, NULL);
+  // .. but a concurrent set may have happened in between our all-clear test and the clearing of the
+  // bit in the mask. We check again to catch this situation.
+  if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) {
+    mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL);
+    return false;
+  }
+  return true;
+}
+
+
+/* --------------------------------------------------------------------------------
+  bitmap
+-------------------------------------------------------------------------------- */
+
+size_t mi_bitmap_size(size_t bit_count, size_t* pchunk_count) {
+  mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0);
+  bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS);
+  mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT);
+  mi_assert_internal(bit_count > 0);
+  const size_t chunk_count = bit_count / MI_BCHUNK_BITS;
+  mi_assert_internal(chunk_count >= 1);
+  const size_t size = offsetof(mi_bitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE);
+  mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 );
+  if (pchunk_count != NULL) { *pchunk_count = chunk_count;  }
+  return size;
+}
+
+
+// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true
+// returns the size of the bitmap
+size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero) {
+  size_t chunk_count;
+  const size_t size = mi_bitmap_size(bit_count, &chunk_count);
+  if (!already_zero) {
+    _mi_memzero_aligned(bitmap, size);
+  }
+  mi_atomic_store_release(&bitmap->chunk_count, chunk_count);
+  mi_assert_internal(mi_atomic_load_relaxed(&bitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT);
+  return size;
+}
+
+
+// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread.
+static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+
+  // start chunk and index
+  size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  const size_t cidx = idx % MI_BCHUNK_BITS;
+  const size_t ccount = _mi_divide_up(n, MI_BCHUNK_BITS);
+
+  // first update the chunkmap
+  mi_bchunk_setN(cmap, chunk_idx, ccount, NULL);
+
+  // first chunk
+  size_t m = MI_BCHUNK_BITS - cidx;
+  if (m > n) { m = n; }
+  mi_bchunk_setN(&chunks[chunk_idx], cidx, m, NULL);
+
+  // n can be large so use memset for efficiency for all in-between chunks
+  chunk_idx++;
+  n -= m;
+  const size_t mid_chunks = n / MI_BCHUNK_BITS;
+  if (mid_chunks > 0) {
+    _mi_memset(&chunks[chunk_idx], ~0, mid_chunks * MI_BCHUNK_SIZE);
+    chunk_idx += mid_chunks;
+    n -= (mid_chunks * MI_BCHUNK_BITS);
+  }
+
+  // last chunk
+  if (n > 0) {
+    mi_assert_internal(n < MI_BCHUNK_BITS);
+    mi_bchunk_setN(&chunks[chunk_idx], 0, n, NULL);
+  }
+}
+
+// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread.
+void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+  mi_assert_internal(idx + n <= mi_bitmap_max_bits(bitmap));
+  mi_bchunks_unsafe_setN(&bitmap->chunks[0], &bitmap->chunkmap, idx, n);
+}
+
+
+
+
+// ------- mi_bitmap_xset ---------------------------------------
+
+// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's).
+bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* palready_set) {
+  mi_assert_internal(n>0);
+  const size_t maxbits = mi_bitmap_max_bits(bitmap);
+  mi_assert_internal(idx + n <= maxbits);
+  if (idx+n > maxbits) { // paranoia
+    if (idx >= maxbits) return false;
+    n = maxbits - idx;
+  }
+
+  // iterate through the chunks
+  size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  size_t cidx = idx % MI_BCHUNK_BITS;
+  bool were_allclear = true;
+  size_t already_set = 0;
+  while (n > 0) {
+    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
+    size_t _already_set = 0;
+    were_allclear = mi_bchunk_setN(&bitmap->chunks[chunk_idx], cidx, m, &_already_set) && were_allclear;
+    already_set += _already_set;
+    mi_bitmap_chunkmap_set(bitmap, chunk_idx); // set afterwards
+    mi_assert_internal(m <= n);
+    n -= m;
+    cidx = 0;
+    chunk_idx++;
+  }
+  if (palready_set != NULL) { *palready_set = already_set;  }
+  return were_allclear;
+}
+
+// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 1's to 0's.
+bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+  const size_t maxbits = mi_bitmap_max_bits(bitmap);
+  mi_assert_internal(idx + n <= maxbits);
+  if (idx+n > maxbits) { // paranoia
+    if (idx >= maxbits) return false;
+    n = maxbits - idx;
+  }
+
+  // iterate through the chunks
+  size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  size_t cidx = idx % MI_BCHUNK_BITS;
+  bool were_allset = true;
+  while (n > 0) {
+    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
+    bool maybe_all_clear = false;
+    were_allset = mi_bchunk_clearN(&bitmap->chunks[chunk_idx], cidx, m, &maybe_all_clear) && were_allset;
+    if (maybe_all_clear) { mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); }
+    mi_assert_internal(m <= n);
+    n -= m;
+    cidx = 0;
+    chunk_idx++;
+  }
+  return were_allset;
+}
+
+// Count bits set in a range of `n` bits.
+size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+  const size_t maxbits = mi_bitmap_max_bits(bitmap);
+  mi_assert_internal(idx + n <= maxbits);
+  if (idx+n > maxbits) { // paranoia
+    if (idx >= maxbits) return 0;
+    n = maxbits - idx;
+  }
+
+  // iterate through the chunks
+  size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  size_t cidx = idx % MI_BCHUNK_BITS;
+  size_t popcount = 0;
+  while (n > 0) {
+    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
+    popcount += mi_bchunk_popcountN(&bitmap->chunks[chunk_idx], cidx, m);
+    mi_assert_internal(m <= n);
+    n -= m;
+    cidx = 0;
+    chunk_idx++;
+  }
+  return popcount;
+}
+
+
+// Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0)
+bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) {
+  return mi_bitmap_setN(bitmap, idx, 1, NULL);
+}
+
+bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx) {
+  return mi_bitmap_clearN(bitmap, idx, 1);
+}
+
+
+
+// ------- mi_bitmap_is_xset ---------------------------------------
+
+// Is a sequence of n bits already all set/cleared?
+bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+  const size_t maxbits = mi_bitmap_max_bits(bitmap);
+  mi_assert_internal(idx + n <= maxbits);
+  if (idx+n > maxbits) { // paranoia
+    if (idx >= maxbits) return false;
+    n = maxbits - idx;
+  }
+
+  // iterate through the chunks
+  size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  size_t cidx = idx % MI_BCHUNK_BITS;
+  bool xset = true;
+  while (n > 0 && xset) {
+    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
+    xset = mi_bchunk_is_xsetN(set, &bitmap->chunks[chunk_idx], cidx, m) && xset;
+    mi_assert_internal(m <= n);
+    n -= m;
+    cidx = 0;
+    chunk_idx++;
+  }
+  return xset;
+}
+
+bool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap) {
+  return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, 0, mi_bitmap_max_bits(bitmap));
+}
+
+/* --------------------------------------------------------------------------------
+  Iterate through a bfield
+-------------------------------------------------------------------------------- */
+
+// Cycle iteration through a bitfield. This is used to space out threads
+// so there is less chance of contention. When searching for a free page we
+// like to first search only the accessed part (so we reuse better). This
+// high point is called the `cycle`.
+//
+// We then iterate through the bitfield as:
+// first: [start, cycle>
+// then : [0, start>
+// then : [cycle, MI_BFIELD_BITS>
+//
+// The start is determined usually as `tseq % cycle` to have each thread
+// start at a different spot.
+// - We use `popcount` to improve branch prediction (maybe not needed? can we simplify?)
+// - The `cycle_mask` is the part `[start, cycle>`.
+#define mi_bfield_iterate(bfield,start,cycle,name_idx,SUF) { \
+  mi_assert_internal(start <= cycle); \
+  mi_assert_internal(start < MI_BFIELD_BITS); \
+  mi_assert_internal(cycle <= MI_BFIELD_BITS); \
+  const mi_bfield_t _cycle_mask##SUF = mi_bfield_mask(cycle - start, start); \
+  size_t _bcount##SUF = mi_bfield_popcount(bfield); \
+  mi_bfield_t _b##SUF = bfield & _cycle_mask##SUF; /* process [start, cycle> first*/\
+  while(_bcount##SUF > 0) { \
+    _bcount##SUF--;\
+    if (_b##SUF==0) { _b##SUF = bfield & ~_cycle_mask##SUF; } /* process [0,start> + [cycle, MI_BFIELD_BITS> next */ \
+    /* size_t name_idx; */ \
+    const bool _found##SUF = mi_bfield_find_least_bit(_b##SUF,&name_idx); \
+    _b##SUF = mi_bfield_clear_least_bit(_b##SUF); /* clear early so `continue` works */ \
+    mi_assert_internal(_found##SUF); MI_UNUSED(_found##SUF); \
+    { \
+
+#define mi_bfield_iterate_end(SUF) \
+    } \
+  } \
+}
+
+
+#define mi_bfield_cycle_iterate(bfield,tseq,cycle,name_idx,SUF) { \
+  const size_t _start##SUF = (uint32_t)(tseq) % (uint32_t)(cycle); /* or: 0 to always search from the start? */\
+  mi_bfield_iterate(bfield,_start##SUF,cycle,name_idx,SUF)
+
+#define mi_bfield_cycle_iterate_end(SUF) \
+  mi_bfield_iterate_end(SUF); \
+}
+
+
+/* --------------------------------------------------------------------------------
+  mi_bitmap_find
+  (used to find free pages)
+-------------------------------------------------------------------------------- */
+
+typedef bool (mi_bitmap_visit_fun_t)(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* idx, void* arg1, void* arg2);
+
+// Go through the bitmap and for every sequence of `n` set bits, call the visitor function.
+// If it returns `true` stop the search.
+static inline bool mi_bitmap_find(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx, mi_bitmap_visit_fun_t* on_find, void* arg1, void* arg2)
+{
+  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
+  for (size_t i = 0; i < chunkmap_max; i++) {
+    // and for each chunkmap entry we iterate over its bits to find the chunks
+    const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
+    size_t hi;
+    if (mi_bfield_find_highest_bit(cmap_entry, &hi)) {
+      size_t eidx = 0;
+      mi_bfield_cycle_iterate(cmap_entry, tseq%8, hi+1, eidx, Y) // reduce the tseq to 8 bins to reduce using extra memory (see `mstress`)
+      {
+        mi_assert_internal(eidx <= MI_BFIELD_BITS);
+        const size_t chunk_idx = i*MI_BFIELD_BITS + eidx;
+        mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));
+        if ((*on_find)(bitmap, chunk_idx, n, pidx, arg1, arg2)) {
+          return true;
+        }
+      }
+      mi_bfield_cycle_iterate_end(Y);
+    }
+  }
+  return false;
+}
+
+
+/* --------------------------------------------------------------------------------
+  Bitmap: try_find_and_claim  -- used to allocate abandoned pages
+  note: the compiler will fully inline the indirect function call
+-------------------------------------------------------------------------------- */
+
+typedef struct mi_claim_fun_data_s {
+  mi_arena_t*   arena;  
+} mi_claim_fun_data_t;
+
+static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* pidx, void* arg1, void* arg2)
+{
+  mi_assert_internal(n==1); MI_UNUSED(n);
+  mi_claim_fun_t* claim_fun = (mi_claim_fun_t*)arg1;
+  mi_claim_fun_data_t* claim_data = (mi_claim_fun_data_t*)arg2;
+  size_t cidx;
+  if mi_likely(mi_bchunk_try_find_and_clear(&bitmap->chunks[chunk_idx], &cidx)) {
+    const size_t slice_index = (chunk_idx * MI_BCHUNK_BITS) + cidx;
+    mi_assert_internal(slice_index < mi_bitmap_max_bits(bitmap));
+    bool keep_set = true;
+    if ((*claim_fun)(slice_index, claim_data->arena, &keep_set)) {
+      // success!
+      mi_assert_internal(!keep_set);
+      *pidx = slice_index;
+      return true;
+    }
+    else {
+      // failed to claim it, set abandoned mapping again (unless the page was freed)
+      if (keep_set) {
+        const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL);
+        mi_assert_internal(wasclear); MI_UNUSED(wasclear);
+      }
+    }
+  }
+  else {
+    // we may find that all are cleared only on a second iteration but that is ok as
+    // the chunkmap is a conservative approximation.
+    mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx);
+  }
+  return false;
+}
+
+// Find a set bit in the bitmap and try to atomically clear it and claim it.
+// (Used to find pages in the pages_abandoned bitmaps.)
+mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,
+  mi_claim_fun_t* claim, mi_arena_t* arena )
+{
+  mi_claim_fun_data_t claim_data = { arena };
+  return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, &claim_data);
+}
+
+
+bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx) {
+  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
+  for (size_t i = chunkmap_max; i > 0; ) {
+    i--;
+    mi_bfield_t cmap = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
+    size_t cmap_idx;
+    if (mi_bsr(cmap,&cmap_idx)) {
+      // highest chunk
+      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
+      size_t cidx;
+      if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) {
+        *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+// Return count of all set bits in a bitmap.
+size_t mi_bitmap_popcount(mi_bitmap_t* bitmap) {
+  // for all chunkmap entries
+  size_t popcount = 0;
+  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
+  for (size_t i = 0; i < chunkmap_max; i++) {
+    mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
+    size_t cmap_idx;
+    // for each chunk (corresponding to a set bit in a chunkmap entry)
+    while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {
+      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
+      // count bits in a chunk
+      popcount += mi_bchunk_popcount(&bitmap->chunks[chunk_idx]);
+    }
+  }
+  return popcount;
+}
+
+
+
+// Clear a bit once it is set.
+void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx) {
+  mi_assert_internal(idx < mi_bitmap_max_bits(bitmap));
+  const size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  const size_t cidx = idx % MI_BCHUNK_BITS;
+  mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));
+  mi_bchunk_clear_once_set(&bitmap->chunks[chunk_idx], cidx);
+}
+
+
+// Visit all set bits in a bitmap.
+// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore?
+bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) {
+  // for all chunkmap entries
+  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
+  for(size_t i = 0; i < chunkmap_max; i++) {
+    mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
+    size_t cmap_idx;
+    // for each chunk (corresponding to a set bit in a chunkmap entry)
+    while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {
+      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
+      // for each chunk field
+      mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];
+      for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
+        const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
+        mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[j]);
+        size_t bidx;
+        while (mi_bfield_foreach_bit(&b, &bidx)) {
+          const size_t idx = base_idx + bidx;
+          if (!visit(idx, 1, arena, arg)) return false;
+        }
+      }
+    }
+  }
+  return true;
+}
+
+// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible.
+// Also clear those ranges atomically.
+// Used by purging to purge larger ranges when possible
+// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore?
+bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) {
+  // for all chunkmap entries
+  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
+  for (size_t i = 0; i < chunkmap_max; i++) {
+    mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
+    size_t cmap_idx;
+    // for each chunk (corresponding to a set bit in a chunkmap entry)
+    while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {
+      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
+      // for each chunk field
+      mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];
+      for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
+        const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
+        mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0);
+        #if MI_DEBUG > 1
+        const size_t bpopcount = mi_popcount(b);
+        size_t rngcount = 0;
+        #endif
+        size_t bidx;
+        while (mi_bfield_find_least_bit(b, &bidx)) {
+          size_t rng = mi_ctz(~(b>>bidx)); // all the set bits from bidx
+          #if MI_DEBUG > 1
+          rngcount += rng;
+          #endif
+          const size_t idx = base_idx + bidx;
+          mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS);
+          mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS);
+          mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap));
+          if (!visit(idx, rng, arena, arg)) {
+            // break early: reset the non-visited bits
+            if (b!=0) {
+              mi_atomic_or_relaxed(&chunk->bfields[j], b);
+            }
+            return false;
+          }
+          // clear rng bits in b
+          b = b & ~mi_bfield_mask(rng, bidx);
+        }
+        mi_assert_internal(rngcount == bpopcount);
+      }
+    }
+  }
+  return true;
+}
+
+// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible,
+// but only in chunks of at least `rngslices` slices (that are also aligned at `rngslices`)
+// and clear those ranges atomically.
+// However, the `rngslices` are capped at `MI_BFIELD_BITS` at most.
+// Used by purging to purge larger ranges when possible. With transparent huge pages we only
+// want to purge whole huge pages (2 MiB) at a time which is what the `rngslices` parameter achieves.
+bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) 
+{
+  // use the generic routine for `rngslices<=1` (as that one finds longest ranges at a time)
+  if (rngslices<=1) {
+    return _mi_bitmap_forall_setc_ranges(bitmap, visit, arena, arg);
+  }
+  // mi_assert_internal(rngslices <= MI_BFIELD_BITS);  
+  if (rngslices > MI_BFIELD_BITS) { rngslices = MI_BFIELD_BITS;  } // cap at MI_BFIELD_BITS at most
+
+  // for all chunkmap entries
+  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
+  for (size_t i = 0; i < chunkmap_max; i++) {
+    mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
+    size_t cmap_idx;
+    // for each chunk (corresponding to a set bit in a chunkmap entry)
+    while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {
+      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
+      // for each chunk field
+      mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];
+      for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
+        const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
+        mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0);   // atomic clear
+        mi_bfield_t skipped = 0;                                                          // but track which bits we skip so we can restore them
+        for(size_t shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) {  // per `rngslices` to keep alignment
+          const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift);
+          if ((b & rngmask) == rngmask) {
+            const size_t idx = base_idx + shift;
+            if (!visit(idx, rngslices, arena, arg)) {
+              // break early: restore non-visited entries
+              mi_bfield_t notyet_visited = 0;
+              if (shift + rngslices < MI_BFIELD_BITS) {
+                notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices)));
+              }
+              mi_assert_internal((notyet_visited & skipped) == 0);
+              if ((notyet_visited | skipped) != 0) {
+                mi_atomic_or_relaxed(&chunk->bfields[j], notyet_visited | skipped);
+              }
+              return false;
+            }
+          }
+          else {
+            skipped = skipped | (b & rngmask);
+          }          
+        } 
+        
+        if (skipped != 0) {
+          mi_atomic_or_relaxed(&chunk->bfields[j], skipped);
+        }
+      }
+    }
+  }
+  return true;
+}
+
+
+/* --------------------------------------------------------------------------------
+  binned bitmap's
+-------------------------------------------------------------------------------- */
+
+
+size_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) {
+  // mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0);
+  bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS);
+  mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT);
+  mi_assert_internal(bit_count > 0);
+  const size_t chunk_count = bit_count / MI_BCHUNK_BITS;
+  mi_assert_internal(chunk_count >= 1);
+  const size_t size = offsetof(mi_bbitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE);
+  mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 );
+  if (pchunk_count != NULL) { *pchunk_count = chunk_count;  }
+  return size;
+}
+
+// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true
+// returns the size of the bitmap
+size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) {
+  size_t chunk_count;
+  const size_t size = mi_bbitmap_size(bit_count, &chunk_count);
+  if (!already_zero) {
+    _mi_memzero_aligned(bbitmap, size);
+  }
+  mi_atomic_store_release(&bbitmap->chunk_count, chunk_count);
+  mi_assert_internal(mi_atomic_load_relaxed(&bbitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT);
+  return size;
+}
+
+void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+  mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap));
+  mi_bchunks_unsafe_setN(&bbitmap->chunks[0], &bbitmap->chunkmap, idx, n);
+}
+
+bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) {
+  const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap);
+  const size_t chunkmap_max = _mi_divide_up(chunk_count, MI_BFIELD_BITS);
+  size_t skip_at_top = chunk_count % MI_BFIELD_BITS;
+  for (size_t i = chunkmap_max; i > 0; ) {
+    i--;
+    mi_bfield_t cmap = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[i]);
+    size_t cmap_idx;
+    // don't consider top 0 bits; set those to 1 here
+    if (skip_at_top > 0) {
+      const size_t mask_top = (~mi_bfield_zero()) << (MI_BFIELD_BITS - skip_at_top);
+      skip_at_top = 0;   // only for the first iteration
+      cmap |= mask_top;
+    }
+    if (mi_bsr(~cmap, &cmap_idx)) {
+      // highest chunk
+      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
+      size_t cidx;
+      if (mi_bchunk_bsr_inv(&bbitmap->chunks[chunk_idx], &cidx)) {
+        *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+
+/* --------------------------------------------------------------------------------
+ binned bitmap used to track free slices
+-------------------------------------------------------------------------------- */
+
+// Assign a specific size bin to a chunk
+static void mi_bbitmap_set_chunk_bin(mi_bbitmap_t* bbitmap, size_t chunk_idx, mi_chunkbin_t bin) {
+  mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap));
+  for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {
+    if (ibin == bin) {
+      const bool was_clear = mi_bchunk_set(& bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);
+      if (was_clear) { mi_os_stat_increase(chunk_bins[ibin],1); }
+    }
+    else {
+      const bool was_set = mi_bchunk_clear(&bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);
+      if (was_set) { mi_os_stat_decrease(chunk_bins[ibin],1); }
+    }
+  }
+}
+
+mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunkmap_t* chunkmap_bins, size_t chunk_idx) {
+  for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {
+    if (mi_bchunk_is_xsetN(MI_BIT_SET, &chunkmap_bins[ibin], chunk_idx, 1)) {
+      return ibin;
+    }
+  }
+  return MI_CBIN_NONE;
+}
+
+// Track the index of the highest chunk that is accessed.
+static void mi_bbitmap_chunkmap_set_max(mi_bbitmap_t* bbitmap, size_t chunk_idx) {
+  size_t oldmax = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed);
+  if mi_unlikely(chunk_idx > oldmax) {
+    mi_atomic_cas_strong_relaxed(&bbitmap->chunk_max_accessed, &oldmax, chunk_idx);
+  }
+}
+
+// Set a bit in the chunkmap
+static void mi_bbitmap_chunkmap_set(mi_bbitmap_t* bbitmap, size_t chunk_idx, bool check_all_set) {
+  mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap));
+  if (check_all_set) {
+    if (mi_bchunk_all_are_set_relaxed(&bbitmap->chunks[chunk_idx])) {
+      // all slices are free in this chunk: return back to the NONE bin
+      mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, MI_CBIN_NONE);
+    }
+  }
+  mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL);
+  mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx);
+}
+
+static bool mi_bbitmap_chunkmap_try_clear(mi_bbitmap_t* bbitmap, size_t chunk_idx) {
+  mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap));
+  // check if the corresponding chunk is all clear
+  if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) return false;
+  // clear the chunkmap bit
+  mi_bchunk_clear(&bbitmap->chunkmap, chunk_idx, NULL);
+  // .. but a concurrent set may have happened in between our all-clear test and the clearing of the
+  // bit in the mask. We check again to catch this situation. (note: mi_bchunk_clear must be acq-rel)
+  if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) {
+    mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL);
+    return false;
+  }
+  mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx);
+  return true;
+}
+
+
+/* --------------------------------------------------------------------------------
+  mi_bbitmap_setN, try_clearN, and is_xsetN
+  (used to find free pages)
+-------------------------------------------------------------------------------- */
+
+// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's).
+bool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+  const size_t maxbits = mi_bbitmap_max_bits(bbitmap);
+  mi_assert_internal(idx + n <= maxbits);
+  if (idx+n > maxbits) { // paranoia
+    if (idx >= maxbits) return false;
+    n = maxbits - idx;
+  }
+
+  // iterate through the chunks
+  size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  size_t cidx = idx % MI_BCHUNK_BITS;
+  bool were_allclear = true;
+  while (n > 0) {
+    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
+    were_allclear = mi_bchunk_setN(&bbitmap->chunks[chunk_idx], cidx, m, NULL) && were_allclear;
+    mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, true); // set afterwards
+    mi_assert_internal(m <= n);
+    n -= m;
+    cidx = 0;
+    chunk_idx++;
+  }
+  return were_allclear;
+}
+
+// ------- mi_bbitmap_try_clearNC ---------------------------------------
+
+// Try to clear `n` bits at `idx` where `n <= MI_BCHUNK_BITS`.
+bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+  mi_assert_internal(n<=MI_BCHUNK_BITS);
+  mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap));
+
+  const size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  const size_t cidx = idx % MI_BCHUNK_BITS;
+  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);  // don't cross chunks (for now)
+  mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap));
+  if (cidx + n > MI_BCHUNK_BITS) return false;
+  bool maybe_all_clear = false;
+  const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear);
+  if (cleared && maybe_all_clear) { mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); }
+  // note: we don't set the size class for an explicit try_clearN (only used by purging)
+  return cleared;
+}
+
+
+
+// ------- mi_bbitmap_is_xset ---------------------------------------
+
+// Is a sequence of n bits already all set/cleared?
+bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+  const size_t maxbits = mi_bbitmap_max_bits(bbitmap);
+  mi_assert_internal(idx + n <= maxbits);
+  if (idx+n > maxbits) { // paranoia
+    if (idx >= maxbits) return false;
+    n = maxbits - idx;
+  }
+
+  // iterate through the chunks
+  size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  size_t cidx = idx % MI_BCHUNK_BITS;
+  bool xset = true;
+  while (n > 0 && xset) {
+    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
+    xset = mi_bchunk_is_xsetN(set, &bbitmap->chunks[chunk_idx], cidx, m) && xset;
+    mi_assert_internal(m <= n);
+    n -= m;
+    cidx = 0;
+    chunk_idx++;
+  }
+  return xset;
+}
+
+
+
+
+/* --------------------------------------------------------------------------------
+  mi_bbitmap_find
+  (used to find free pages)
+-------------------------------------------------------------------------------- */
+
+typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx);
+
+// Go through the bbitmap and for every sequence of `n` set bits, call the visitor function.
+// If it returns `true` stop the search.
+//
+// This is used for finding free blocks and it is important to be efficient (with 2-level bitscan)
+// but also reduce fragmentation (through size bins).
+static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find)
+{
+  // we space out threads to reduce contention
+  const size_t cmap_max_count  = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS);
+  const size_t chunk_acc       = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed);
+  const size_t cmap_acc        = chunk_acc / MI_BFIELD_BITS;
+  const size_t cmap_acc_bits   = 1 + (chunk_acc % MI_BFIELD_BITS);
+
+  // create a mask over the chunkmap entries to iterate over them efficiently
+  mi_assert_internal(MI_BFIELD_BITS >= MI_BCHUNK_FIELDS);
+  const mi_bfield_t cmap_mask  = mi_bfield_mask(cmap_max_count,0);
+  const size_t cmap_cycle      = cmap_acc+1;
+  const mi_chunkbin_t bbin = mi_chunkbin_of(n);
+  // visit each cmap entry
+  size_t cmap_idx = 0;
+  mi_bfield_cycle_iterate(cmap_mask, tseq, cmap_cycle, cmap_idx, X)
+  {
+    // and for each chunkmap entry we iterate over its bits to find the chunks
+    const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[cmap_idx]);
+    const size_t cmap_entry_cycle = (cmap_idx != cmap_acc ? MI_BFIELD_BITS : cmap_acc_bits);
+    if (cmap_entry == 0) {
+      continue;
+    }
+
+    // get size bin masks
+    mi_bfield_t cmap_bins[MI_CBIN_COUNT] = { 0 };
+    cmap_bins[MI_CBIN_NONE] = cmap_entry;
+    for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {
+      const mi_bfield_t cmap_bin = mi_atomic_load_relaxed(&bbitmap->chunkmap_bins[ibin].bfields[cmap_idx]);
+      cmap_bins[ibin] = cmap_bin & cmap_entry;
+      cmap_bins[MI_CBIN_NONE] &= ~cmap_bin;      // clear bits that are in an assigned size bin
+    }
+
+    // consider only chunks for a particular size bin at a time
+    // this picks the best bin only within a cmap entry (~ 1GiB address space), but avoids multiple
+    // iterations through all entries.
+    mi_assert_internal(bbin < MI_CBIN_NONE);
+    for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin <= MI_CBIN_NONE;
+          // skip from bbin to NONE (so, say, a SMALL will never be placed in a OTHER, MEDIUM, or LARGE chunk to reduce fragmentation)
+          ibin = (ibin == bbin ? MI_CBIN_NONE : mi_chunkbin_inc(ibin)))
+    {
+      mi_assert_internal(ibin < MI_CBIN_COUNT);
+      const mi_bfield_t cmap_bin = cmap_bins[ibin];
+      size_t eidx = 0;
+      mi_bfield_cycle_iterate(cmap_bin, tseq, cmap_entry_cycle, eidx, Y)
+      {
+        // assertion doesn't quite hold as the max_accessed may be out-of-date
+        // mi_assert_internal(cmap_entry_cycle > eidx || ibin == MI_CBIN_NONE);
+
+        // get the chunk
+        const size_t chunk_idx = cmap_idx*MI_BFIELD_BITS + eidx;
+        mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx];
+
+        size_t cidx;
+        if ((*on_find)(chunk, n, &cidx)) {
+          if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin
+            // this chunk is now reserved for the `bbin` size class
+            mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin);
+          }
+          *pidx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
+          mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap));
+          return true;
+        }
+        else {
+          // todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?`
+          // we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation.
+          mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);
+        }
+      }
+      mi_bfield_cycle_iterate_end(Y);
+    }
+  }
+  mi_bfield_cycle_iterate_end(X);
+  return false;
+}
+
+/* --------------------------------------------------------------------------------
+  mi_bbitmap_try_find_and_clear -- used to find free pages
+  note: the compiler will fully inline the indirect function calls
+-------------------------------------------------------------------------------- */
+
+bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) {
+  return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 1, pidx, &mi_bchunk_try_find_and_clear_1);
+}
+
+bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) {
+  return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 8, pidx, &mi_bchunk_try_find_and_clear_8);
+}
+
+// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) {
+//   return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, MI_BFIELD_BITS, pidx, &mi_bchunk_try_find_and_clear_X);
+// }
+
+bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
+  mi_assert_internal(n<=MI_BFIELD_BITS);
+  return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNX);
+}
+
+bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
+  mi_assert_internal(n<=MI_BCHUNK_BITS);
+  return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNC);
+}
+
+
+/* --------------------------------------------------------------------------------
+  mi_bbitmap_try_find_and_clear for huge objects spanning multiple chunks
+-------------------------------------------------------------------------------- */
+
+// Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunks
+static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) {
+  mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap));
+
+  size_t m = n;      // bits to go
+  size_t count = 0;  // chunk count
+  while (m > 0) {
+    mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];
+    if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL)) {
+      goto rollback;
+    }
+    m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS);
+    count++;
+  }
+  return true;
+
+rollback:
+  // we only need to reset chunks the we just fully cleared
+  while (count > 0) {
+    count--;
+    mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];
+    mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL);
+  }
+  return false;
+}
+
+// Go through the bbitmap to find a sequence of `n` bits and clear them atomically where `n > MI_ARENA_MAX_CHUNK_OBJ_SIZE`
+// Since these are very large object allocations we always search from the start and only consider starting at the start
+// of a chunk (for fragmentation and efficiency).
+// Todo: for now we try to find full empty chunks to cover `n` but we can allow a partial chunk at the end
+// Todo: This scans directly through the chunks -- we might want to consult the cmap as well?
+bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
+  MI_UNUSED(tseq);
+  mi_assert(n > 0); if (n==0) { return false; }
+
+  const size_t chunk_max = mi_bbitmap_chunk_count(bbitmap);
+  const size_t chunk_req = _mi_divide_up(n, MI_BCHUNK_BITS);  // minimal number of chunks needed
+  if (chunk_max < chunk_req) { return false; }
+
+  // iterate through the chunks
+  size_t chunk_idx = 0;
+  while (chunk_idx <= chunk_max - chunk_req)
+  {
+    size_t count = 0;  // chunk count
+    do {
+      mi_assert_internal(chunk_idx + count < chunk_max);
+      mi_bchunk_t* const chunk = &bbitmap->chunks[chunk_idx + count];
+      if (!mi_bchunk_all_are_set_relaxed(chunk)) {
+        break;
+      }
+      else {
+        count++;
+      }
+    }
+    while (count < chunk_req);
+
+    // did we find a suitable range?
+    if (count == chunk_req) {
+      // now try to claim it!
+      if (mi_bchunk_try_clearN_(bbitmap, chunk_idx, n)) {
+        *pidx = (chunk_idx * MI_BCHUNK_BITS);
+        for (size_t i = 0; i < count; i++) {
+          mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx + i, MI_CBIN_HUGE);
+        }
+        mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap));
+        return true;
+      }
+    }
+
+    // keep searching but skip the scanned range
+    chunk_idx += count+1;
+  }
+  return false;
+}
+
+
+
+
+
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.h
new file mode 100644
index 00000000..ca51a4e0
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.h
@@ -0,0 +1,343 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+/* ----------------------------------------------------------------------------
+Concurrent bitmap that can set/reset sequences of bits atomically
+---------------------------------------------------------------------------- */
+#pragma once
+#ifndef MI_BITMAP_H
+#define MI_BITMAP_H
+
+/* --------------------------------------------------------------------------------
+  Atomic bitmaps with release/acquire guarantees:
+
+  `mi_bfield_t`: is a single machine word that can efficiently be bit counted (usually `size_t`)
+      each bit usually represents a single MI_ARENA_SLICE_SIZE in an arena (64 KiB).
+      We need 16K bits to represent a 1GiB arena.
+
+  `mi_bchunk_t`: a chunk of bfield's of a total of MI_BCHUNK_BITS (= 512 on 64-bit, 256 on 32-bit)
+      allocations never span across chunks -- so MI_ARENA_MAX_OBJ_SIZE is the number
+      of bits in a chunk times the MI_ARENA_SLICE_SIZE (512 * 64KiB = 32 MiB).
+      These chunks are cache-aligned and we can use AVX2/AVX512/NEON/SVE/SVE2/etc. instructions
+      to scan for bits (perhaps) more efficiently.
+
+      We allocate byte-sized ranges aligned to bytes in the bfield, and bfield-sized
+      ranges aligned to a bfield.
+
+    Searching linearly through the chunks would be too slow (16K bits per GiB).
+    Instead we add a "chunkmap" to do a two-level search (more or less a btree of depth 2).
+
+   `mi_bchunkmap_t` (== `mi_bchunk_t`): for each chunk we track if it has (potentially) any bit set.
+      The chunkmap has 1 bit per chunk that is set if the chunk potentially has a bit set.
+      This is used to avoid scanning every chunk. (and thus strictly an optimization)
+      It is conservative: it is fine to set a bit in the chunk map even if the chunk turns out
+      to have no bits set. It is also allowed to briefly have a clear bit even if the
+      chunk has bits set -- as long as we guarantee that the bit will be set later on;
+      (this allows us to set the chunkmap bit right after we set a bit in the corresponding chunk).
+
+      However, when we clear a bit in a chunk, and the chunk is indeed all clear, we
+      cannot safely clear the bit corresponding to the chunk in the chunkmap since it
+      may race with another thread setting a bit in the same chunk. Therefore, when
+      clearing, we first test if a chunk is clear, then clear the chunkmap bit, and
+      then test again to catch any set bits that we may have missed.
+
+      Since the chunkmap may thus be briefly out-of-sync, this means that we may sometimes
+      not find a free page even though it's there (but we accept this as we avoid taking
+      full locks). (Another way to do this is to use an epoch but we like to avoid that complexity
+      for now).
+
+   `mi_bitmap_t`: a bitmap with N chunks. A bitmap has a chunkmap of MI_BCHUNK_BITS (512)
+      and thus has at most 512 chunks (=2^18 bits x 64 KiB slices = 16 GiB max arena size).
+      The minimum is 1 chunk which is a 32 MiB arena.
+
+   For now, the implementation assumes MI_HAS_FAST_BITSCAN and uses trailing-zero-count
+   and pop-count (but we think it can be adapted work reasonably well on older hardware too)
+--------------------------------------------------------------------------------------------- */
+
+// A word-size bit field.
+typedef size_t mi_bfield_t;
+
+#define MI_BFIELD_BITS_SHIFT         (MI_SIZE_SHIFT+3)
+#define MI_BFIELD_BITS               (1 << MI_BFIELD_BITS_SHIFT)
+#define MI_BFIELD_SIZE               (MI_BFIELD_BITS/8)
+#define MI_BFIELD_LO_BIT8            (((~(mi_bfield_t)0))/0xFF)         // 0x01010101 ..
+#define MI_BFIELD_HI_BIT8            (MI_BFIELD_LO_BIT8 << 7)           // 0x80808080 ..
+
+#define MI_BCHUNK_SIZE               (MI_BCHUNK_BITS / 8)
+#define MI_BCHUNK_FIELDS             (MI_BCHUNK_BITS / MI_BFIELD_BITS)  // 8 on both 64- and 32-bit
+
+
+// some compiler (msvc in C mode) cannot have expressions in the alignment attribute
+#if MI_BCHUNK_SIZE==64
+#define mi_decl_bchunk_align  mi_decl_align(64)
+#elif MI_BCHUNK_SIZE==32
+#define mi_decl_bchunk_align  mi_decl_align(32)
+#else
+#define mi_decl_bchunk_align  mi_decl_align(MI_BCHUNK_SIZE)
+#endif
+
+
+// A bitmap chunk contains 512 bits on 64-bit  (256 on 32-bit)
+typedef mi_decl_bchunk_align struct mi_bchunk_s {
+  _Atomic(mi_bfield_t) bfields[MI_BCHUNK_FIELDS];
+} mi_bchunk_t;
+
+
+// The chunkmap has one bit per corresponding chunk that is set if the chunk potentially has bits set.
+// The chunkmap is itself a chunk.
+typedef mi_bchunk_t mi_bchunkmap_t;
+
+#define MI_BCHUNKMAP_BITS             MI_BCHUNK_BITS
+
+#define MI_BITMAP_MAX_CHUNK_COUNT     (MI_BCHUNKMAP_BITS)
+#define MI_BITMAP_MIN_CHUNK_COUNT     (1)
+#if MI_SIZE_BITS > 32
+#define MI_BITMAP_DEFAULT_CHUNK_COUNT     (64)  // 2 GiB on 64-bit -- this is for the page map
+#else
+#define MI_BITMAP_DEFAULT_CHUNK_COUNT      (1)
+#endif
+#define MI_BITMAP_MAX_BIT_COUNT       (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS)  // 16 GiB arena
+#define MI_BITMAP_MIN_BIT_COUNT       (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS)  // 32 MiB arena
+#define MI_BITMAP_DEFAULT_BIT_COUNT   (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS)  // 2 GiB arena
+
+
+// An atomic bitmap
+typedef mi_decl_bchunk_align struct mi_bitmap_s {
+  _Atomic(size_t)  chunk_count;         // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)
+  size_t           _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 1];    // suppress warning on msvc
+  mi_bchunkmap_t   chunkmap;
+  mi_bchunk_t      chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT];        // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT
+} mi_bitmap_t;
+
+
+static inline size_t mi_bitmap_chunk_count(const mi_bitmap_t* bitmap) {
+  return mi_atomic_load_relaxed(&((mi_bitmap_t*)bitmap)->chunk_count);
+}
+
+static inline size_t mi_bitmap_max_bits(const mi_bitmap_t* bitmap) {
+  return (mi_bitmap_chunk_count(bitmap) * MI_BCHUNK_BITS);
+}
+
+
+
+/* --------------------------------------------------------------------------------
+  Atomic bitmap operations
+-------------------------------------------------------------------------------- */
+
+// Many operations are generic over setting or clearing the bit sequence: we use `mi_xset_t` for this (true if setting, false if clearing)
+typedef bool  mi_xset_t;
+#define MI_BIT_SET    (true)
+#define MI_BIT_CLEAR  (false)
+
+
+// Required size of a bitmap to represent `bit_count` bits.
+size_t mi_bitmap_size(size_t bit_count, size_t* chunk_count);
+
+// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true
+// returns the size of the bitmap.
+size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero);
+
+// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).
+// Not atomic so only use if still local to a thread.
+void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n);
+
+
+// Set a bit in the bitmap; returns `true` if it atomically transitioned from 0 to 1
+bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx);
+
+// Clear a bit in the bitmap; returns `true` if it atomically transitioned from 1 to 0
+bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx);
+
+// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 0's to 1's
+// If `already_set` is not NULL, it is set to count of bits were already all set.
+// (this is used for correct statistics if commiting over a partially committed area)
+bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_set);
+
+// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 1's to 0's
+bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n);
+
+
+// Is a sequence of n bits already all set/cleared?
+bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n);
+
+// Is the bitmap completely clear?
+bool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap);
+
+// Is a sequence of n bits already set?
+// (Used to check if a memory range is already committed)
+static inline bool mi_bitmap_is_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
+  return mi_bitmap_is_xsetN(MI_BIT_SET, bitmap, idx, n);
+}
+
+// Is a sequence of n bits already clear?
+static inline bool mi_bitmap_is_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
+  return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, idx, n);
+}
+
+static inline bool mi_bitmap_is_set(mi_bitmap_t* bitmap, size_t idx) {
+  return mi_bitmap_is_setN(bitmap, idx, 1);
+}
+
+static inline bool mi_bitmap_is_clear(mi_bitmap_t* bitmap, size_t idx) {
+  return mi_bitmap_is_clearN(bitmap, idx, 1);
+}
+
+// Called once a bit is cleared to see if the memory slice can be claimed.
+typedef bool (mi_claim_fun_t)(size_t slice_index, mi_arena_t* arena, bool* keep_set);
+
+// Find a set bits in the bitmap, atomically clear it, and check if `claim` returns true.
+// If not claimed, continue on (potentially setting the bit again depending on `keep_set`).
+// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
+mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,
+                                                    mi_claim_fun_t* claim, mi_arena_t* arena );
+
+
+// Atomically clear a bit but only if it is set. Will block otherwise until the bit is set.
+// This is used to delay free-ing a page that it at the same time being considered to be
+// allocated from `mi_arena_try_abandoned` (and is in the `claim` function of `mi_bitmap_try_find_and_claim`).
+void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx);
+
+
+// If a bit is set in the bitmap, return `true` and set `idx` to the index of the highest bit.
+// Otherwise return `false` (and `*idx` is undefined).
+// Used for unloading arena's
+bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx);
+
+// Return count of all set bits in a bitmap.
+size_t mi_bitmap_popcount(mi_bitmap_t* bitmap);
+
+
+typedef bool (mi_forall_set_fun_t)(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg2);
+
+// Visit all set bits in a bitmap (`slice_count == 1`)
+bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
+
+// Visit all set bits in a bitmap with larger ranges if possible (`slice_count >= 1`)
+// Ranges will never cross chunk boundaries though (and `slice_count <= MI_BCHUNK_BITS`)
+bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
+
+// Visit all set bits in a bitmap with at least `rngslices` at a time (and aligned to `rngslices`). 
+// This is used by purging to not break up transparent huge pages for example.
+// Ranges will never cross chunk boundaries (and `slice_count <= MI_BCHUNK_BITS`).
+bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
+
+// Count all set bits in given range in the bitmap.
+size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n);
+
+/* ----------------------------------------------------------------------------
+  Binned concurrent bitmap
+  Assigns a size class to each chunk such that small blocks don't cause too
+  much fragmentation since we keep chunks for larger blocks separate.
+---------------------------------------------------------------------------- */
+
+// mi_chunkbin_t is defined in mimalloc-stats.h
+
+static inline mi_chunkbin_t mi_chunkbin_inc(mi_chunkbin_t bbin) {
+  mi_assert_internal(bbin < MI_CBIN_COUNT);
+  return (mi_chunkbin_t)((int)bbin + 1);
+}
+
+static inline mi_chunkbin_t mi_chunkbin_dec(mi_chunkbin_t bbin) {
+  mi_assert_internal(bbin > MI_CBIN_NONE);
+  return (mi_chunkbin_t)((int)bbin - 1);
+}
+
+static inline mi_chunkbin_t mi_chunkbin_of(size_t slice_count) {
+  if (slice_count==1) return MI_CBIN_SMALL;
+  if (slice_count==8) return MI_CBIN_MEDIUM;
+  #if MI_ENABLE_LARGE_PAGES
+  if (slice_count==MI_BFIELD_BITS) return MI_CBIN_LARGE;
+  #endif
+  if (slice_count > MI_BCHUNK_BITS) return MI_CBIN_HUGE;
+  return MI_CBIN_OTHER;
+}
+
+// An atomic "binned" bitmap for the free slices where we keep chunks reserved for particular size classes
+typedef mi_decl_bchunk_align struct mi_bbitmap_s {
+  _Atomic(size_t)  chunk_count;         // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)
+  _Atomic(size_t)  chunk_max_accessed;  // max chunk index that was once cleared or set
+  #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 2
+  size_t           _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 2];    // suppress warning on msvc by aligning manually
+  #endif
+  mi_bchunkmap_t   chunkmap;
+  mi_bchunkmap_t   chunkmap_bins[MI_CBIN_COUNT - 1];             // chunkmaps with bit set if the chunk is in that size class (excluding MI_CBIN_NONE)
+  mi_bchunk_t      chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT];        // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT
+} mi_bbitmap_t;
+
+
+static inline size_t mi_bbitmap_chunk_count(const mi_bbitmap_t* bbitmap) {
+  return mi_atomic_load_relaxed(&((mi_bbitmap_t*)bbitmap)->chunk_count);
+}
+
+static inline size_t mi_bbitmap_max_bits(const mi_bbitmap_t* bbitmap) {
+  return (mi_bbitmap_chunk_count(bbitmap) * MI_BCHUNK_BITS);
+}
+
+mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunk_t* chunkmap_bins, size_t chunk_idx);
+
+size_t mi_bbitmap_size(size_t bit_count, size_t* chunk_count);
+
+// If a bit is clear in the bitmap, return `true` and set `idx` to the index of the highest bit that was clear.
+// Otherwise return `false` (and `*idx` is undefined).
+// Used for debug output.
+bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx);
+
+// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true
+// returns the size of the bitmap.
+size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero);
+
+// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).
+// Not atomic so only use if still local to a thread.
+void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
+
+
+// Set a sequence of `n` bits in the bbitmap; returns `true` if atomically transitioned from all 0's to 1's
+bool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
+
+
+// Is a sequence of n bits already all set/cleared?
+bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n);
+
+// Is a sequence of n bits already set?
+// (Used to check if a memory range is already committed)
+static inline bool mi_bbitmap_is_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
+  return mi_bbitmap_is_xsetN(MI_BIT_SET, bbitmap, idx, n);
+}
+
+// Is a sequence of n bits already clear?
+static inline bool mi_bbitmap_is_clearN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
+  return mi_bbitmap_is_xsetN(MI_BIT_CLEAR, bbitmap, idx, n);
+}
+
+
+// Try to atomically transition `n` bits from all set to all clear. Returns `true` on succes.
+// `n` cannot cross chunk boundaries, where `n <= MI_CHUNK_BITS`.
+bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
+
+
+// Specialized versions for common bit sequence sizes
+bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx);  // 1-bit
+bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 8-bits
+// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // MI_BFIELD_BITS
+bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // < MI_BFIELD_BITS
+bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS
+bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BCHUNK_BITS
+
+// Find a sequence of `n` bits in the bbitmap with all bits set, and try to atomically clear all.
+// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
+mi_decl_nodiscard static inline bool mi_bbitmap_try_find_and_clearN(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
+  if (n==1) return mi_bbitmap_try_find_and_clear(bbitmap, tseq, pidx);               // small pages
+  if (n==8) return mi_bbitmap_try_find_and_clear8(bbitmap, tseq, pidx);              // medium pages
+  // if (n==MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearX(bbitmap, tseq, pidx); // large pages
+  if (n==0) return false;
+  if (n<=MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearNX(bbitmap, tseq, n, pidx);
+  if (n<=MI_BCHUNK_BITS) return mi_bbitmap_try_find_and_clearNC(bbitmap, tseq, n, pidx);
+  return mi_bbitmap_try_find_and_clearN_(bbitmap, tseq, n, pidx);
+}
+
+
+#endif // MI_BITMAP_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/free.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/free.c
new file mode 100644
index 00000000..6f33340d
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/free.c
@@ -0,0 +1,658 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#if !defined(MI_IN_ALLOC_C)
+#error "this file should be included from 'alloc.c' (so aliases can work from alloc-override)"
+// add includes help an IDE
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"   // _mi_prim_thread_id()
+#endif
+
+// forward declarations
+static void   mi_check_padding(const mi_page_t* page, const mi_block_t* block);
+static bool   mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block);
+static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded);
+static void   mi_stat_free(const mi_page_t* page, const mi_block_t* block);
+
+
+// ------------------------------------------------------
+// Free
+// ------------------------------------------------------
+
+// regular free of a (thread local) block pointer
+// fast path written carefully to prevent spilling on the stack
+static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full)
+{
+  MI_UNUSED(was_guarded);
+  // checks  
+  if mi_unlikely(mi_check_is_double_free(page, block)) return;
+  if (!was_guarded) { mi_check_padding(page, block); }
+  if (track_stats) { mi_stat_free(page, block); }
+  #if (MI_DEBUG>0) && !MI_TRACK_ENABLED  && !MI_TSAN
+  memset(block, MI_DEBUG_FREED, mi_page_block_size(page));
+  #endif
+  if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned
+
+  // actual free: push on the local free list
+  mi_block_set_next(page, block, page->local_free);
+  page->local_free = block;
+  if mi_unlikely(--page->used == 0) {
+    if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page)
+      _mi_page_retire(page); 
+    }
+  }
+  else if mi_unlikely(check_full && mi_page_is_in_full(page)) {
+    _mi_page_unfull(page);
+  }
+}
+
+// Forward declaration for multi-threaded collect
+static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept;
+
+// Free a block multi-threaded
+static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded) mi_attr_noexcept
+{
+  MI_UNUSED(was_guarded);
+  // adjust stats (after padding check and potentially recursive `mi_free` above)
+  mi_stat_free(page, block);    // stat_free may access the padding
+  mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded));
+
+  // _mi_padding_shrink(page, block, sizeof(mi_block_t));
+#if (MI_DEBUG>0) && !MI_TRACK_ENABLED  && !MI_TSAN       // note: when tracking, cannot use mi_usable_size with multi-threading
+  if (!was_guarded) {
+    size_t dbgsize = mi_usable_size(block);
+    if (dbgsize > MI_MiB) { dbgsize = MI_MiB; }
+    _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);
+  }
+#endif
+
+  // push atomically on the page thread free list
+  mi_thread_free_t tf_new;
+  mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);
+  do {
+    mi_block_set_next(page, block, mi_tf_block(tf_old));
+    tf_new = mi_tf_create(block, true /* always use owned: try to claim it if the page is abandoned */);
+  } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough?
+
+  // and atomically try to collect the page if it was abandoned
+  const bool is_owned_now = !mi_tf_is_owned(tf_old);
+  if (is_owned_now) {
+    mi_assert_internal(mi_page_is_abandoned(page));
+    mi_free_try_collect_mt(page,block);
+  }
+}
+
+
+// Adjust a block that was allocated aligned, to the actual start of the block in the page.
+// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the
+// `page_start` and `block_size` fields; however these are constant and the page won't be
+// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently.
+mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
+  mi_assert_internal(page!=NULL && p!=NULL);
+
+  const size_t diff = (uint8_t*)p - mi_page_start(page);
+  const size_t block_size = mi_page_block_size(page);
+  const size_t adjust = (_mi_is_power_of_two(block_size) ? diff & (block_size - 1) : diff % block_size);
+  return (mi_block_t*)((uintptr_t)p - adjust);
+}
+
+// forward declaration for a MI_GUARDED build
+#if MI_GUARDED
+static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration
+static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
+  if (mi_block_ptr_is_guarded(block, p)) { 
+    mi_block_unguard(page, block, p); 
+    return true;
+  }
+  else {
+    return false;
+  }
+}
+#else
+static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
+  MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);
+  return false;
+}
+#endif
+
+static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, void* p ) {
+  mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer
+  #if MI_SECURE > 0
+  // in secure mode we always unalign to guard against free-ing interior pointers
+  return _mi_page_ptr_unalign(page,p);
+  #else
+  MI_UNUSED(page);
+  return (mi_block_t*)p;
+  #endif
+}
+
+
+// free a local pointer  (page parameter comes first for better codegen)
+static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept {
+  mi_assert_internal(p!=NULL && page != NULL);
+  mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
+  const bool was_guarded = mi_block_check_unguard(page, block, p);
+  mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */);
+}
+
+// free a pointer owned by another thread (page parameter comes first for better codegen)
+static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p) mi_attr_noexcept {
+  mi_assert_internal(p!=NULL && page != NULL);
+  mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
+  const bool was_guarded = mi_block_check_unguard(page, block, p);
+  mi_free_block_mt(page, block, was_guarded);
+}
+
+// generic free (for runtime integration)
+void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {
+  if (is_local) mi_free_generic_local(page,p);
+           else mi_free_generic_mt(page,p);
+}
+
+
+// Get the page belonging to a pointer
+// Does further checks in debug mode to see if this was a valid pointer.
+static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg)
+{
+  MI_UNUSED_RELEASE(msg);
+  #if MI_DEBUG
+  if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) {
+    _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p);
+    return NULL;
+  }
+  mi_page_t* page = _mi_safe_ptr_page(p);
+  if (p != NULL && page == NULL) {
+    _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p);
+  }
+  return page;
+  #else
+  return _mi_ptr_page(p);
+  #endif
+}
+
+// Free a block
+// Fast path written carefully to prevent register spilling on the stack
+static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* page)  
+{
+  if mi_unlikely(page==NULL) return;  // page will be NULL if p==NULL
+  mi_assert_internal(p!=NULL && page!=NULL);
+  if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
+
+  const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page));
+  if mi_likely(xtid == 0) {                        // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0`
+    // thread-local, aligned, and not a full page
+    mi_block_t* const block = mi_validate_block_from_ptr(page,p);
+    mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */);
+  }
+  else if (xtid <= MI_PAGE_FLAG_MASK) {            // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0`
+    // page is local, but is full or contains (inner) aligned blocks; use generic path
+    mi_free_generic_local(page, p);
+  }
+  // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap)
+  else if ((xtid & MI_PAGE_FLAG_MASK) == 0) {      // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0`
+    // blocks are aligned (and not a full page); push on the thread_free list
+    mi_block_t* const block = mi_validate_block_from_ptr(page,p);
+    mi_free_block_mt(page,block,false /* was_guarded */);
+  }
+  else {
+    // page is full or contains (inner) aligned blocks; use generic multi-thread path
+    mi_free_generic_mt(page, p);
+  }
+}
+
+void mi_free(void* p) mi_attr_noexcept {
+  mi_page_t* const page = mi_validate_ptr_page(p,"mi_free");  
+  mi_free_ex(p, NULL, page);
+}
+
+void mi_ufree(void* p, size_t* usable) mi_attr_noexcept {
+  mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree");  
+  mi_free_ex(p, usable, page);
+}
+
+void mi_free_small(void* p) mi_attr_noexcept {
+  // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`.
+  // If we keep page info in front of the page area for small objects, we can find the info
+  // just by aligning down the pointer instead of looking it up in the page map.
+  #if MI_PAGE_META_ALIGNED_FREE_SMALL 
+    #if MI_GUARDED 
+    #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as MI_GUARDED is defined"
+    mi_free(p);
+    #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE
+    #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size"
+    mi_free(p);
+    #else
+      mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE);
+      mi_assert(page == mi_validate_ptr_page(p,"mi_free_small"));
+      mi_assert((void*)page == _mi_align_down_ptr(page->page_start,MI_SMALL_PAGE_SIZE));
+      mi_assert(page->block_size <= MI_SMALL_SIZE_MAX);  // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small`
+      mi_free_ex(p, NULL, page);
+    #endif
+  #else
+  mi_free(p);
+  #endif  
+}
+
+
+// --------------------------------------------------------------------------------------------
+// `mi_free_try_collect_mt`: Potentially collect a page in a free in an abandoned page.
+// 1. if the page becomes empty, free it
+// 2. if it can be reclaimed, reclaim it in our theap
+// 3. if it went to < 7/8th used, re-abandon to be mapped (so it can be found by theaps looking for free pages)
+// --------------------------------------------------------------------------------------------
+
+// Helper for mi_free_try_collect_mt: free if the page has no more used blocks (this is updated by `_mi_page_free_collect(_partly)`)
+static bool mi_abandoned_page_try_free(mi_page_t* page)
+{
+  if (!mi_page_all_free(page)) return false;
+  // first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish)
+  _mi_arenas_page_unabandon(page,NULL);
+  _mi_arenas_page_free(page,NULL); // we can now free the page directly
+  return true;
+}
+
+// Helper for mi_free_try_collect_mt: try if we can reabandon a previously abandoned mostly full page to be mapped
+static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page)
+{
+  // if the page is unmapped, try to reabandon so it can possibly be mapped and found for allocations
+  // We only reabandon if a full page starts to have enough blocks available to prevent immediate re-abandon of a full page
+  if (mi_page_is_mostly_used(page)) return false;   // not too full
+  if (page->memid.memkind != MI_MEM_ARENA || mi_page_is_abandoned_mapped(page)) return false;  // and not already mapped (or unmappable)
+
+  mi_assert(!mi_page_is_full(page));
+  return _mi_arenas_page_try_reabandon_to_mapped(page);
+}
+
+// Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently
+// freed in the meantime. Returns `true` if the page was freed.
+// By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`.
+static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) {
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(!mi_page_all_free(page));
+  // try to cas atomically the original free list (`mt_free`) back with the ownership cleared.
+  mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true);
+  mi_thread_free_t tf_new    = mi_tf_create(expected_thread_free, false);
+  while mi_unlikely(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_expect, tf_new)) {
+    mi_assert_internal(mi_tf_is_owned(tf_expect));
+    // while the xthread_free list is not empty..
+    while (mi_tf_block(tf_expect) != NULL) {
+      // if there were concurrent updates to the thread-free list, we retry to free or reabandon to mapped (if it became !mosty_used).
+      _mi_page_free_collect(page,false);  // update used count
+      if (mi_abandoned_page_try_free(page)) return;
+      if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
+      // otherwise continue un-owning
+      tf_expect = mi_atomic_load_relaxed(&page->xthread_free);
+    }
+    // and try again to release ownership
+    mi_assert_internal(mi_tf_block(tf_expect)==NULL);
+    tf_new = mi_tf_create(NULL, false);
+  }
+}
+
+static inline bool mi_page_queue_len_is_atmost( mi_theap_t* theap, size_t block_size, long atmost) {
+  if (atmost < 0) return false;
+  mi_page_queue_t* const pq = mi_page_queue(theap,block_size);
+  mi_assert_internal(pq!=NULL);
+  return (pq->count <= (size_t)atmost);
+}
+
+// Helper for mi_free_try_collect_mt:  try to reclaim the page for ourselves
+static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long reclaim_on_free) mi_attr_noexcept
+{
+  // note: reclaiming can improve benchmarks like `larson` or `rbtree-ck` a lot even in the single-threaded case,
+  // since free-ing from an owned page avoids atomic operations. However, if we reclaim too eagerly in
+  // a multi-threaded scenario we may start to hold on to too much memory and reduce reuse among threads.
+  // If the current theap is where the page originally came from, we reclaim much more eagerly while
+  // 'cross-thread' reclaiming on free is by default off (and we only 'reclaim' these by finding the abandoned
+  // pages when we allocate a fresh page).
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(!mi_page_all_free(page));
+  mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE);
+  mi_assert_internal(reclaim_on_free >= 0);
+
+  // dont reclaim if we just have terminated this thread and we should
+  // not reinitialize the theap for this thread. (can happen due to thread-local destructors for example -- issue #944)
+  if (!_mi_thread_is_initialized()) return false;
+
+  // get our theap 
+  mi_theap_t* const theap = _mi_page_associated_theap_peek(page);
+  if (theap==NULL || !theap->allow_page_reclaim) return false;
+  
+  // todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance?
+  // set max_reclaim limit
+  long max_reclaim = 0;
+  if mi_likely(theap == page->theap) {  // did this page originate from the current theap? (and thus allocated from this thread)
+    // originating theap
+    max_reclaim = _mi_option_get_fast(theap->tld->is_in_threadpool ? mi_option_page_cross_thread_max_reclaim : mi_option_page_max_reclaim);
+  }
+  else if (reclaim_on_free == 1 &&               // if cross-thread is allowed
+            !theap->tld->is_in_threadpool &&      // and we are not part of a threadpool
+            !mi_page_is_mostly_used(page) &&     // and the page is not too full
+            _mi_arena_memid_is_suitable(page->memid, _mi_theap_heap(theap)->exclusive_arena)) {   // and it fits our memory
+    // across threads
+    max_reclaim = _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim);
+  }
+
+  // are we within the reclaim limit?
+  if (max_reclaim >= 0 && !mi_page_queue_len_is_atmost(theap, page->block_size, max_reclaim)) {
+    return false;
+  }
+
+  // reclaim the page into this theap
+  // first remove it from the abandoned pages in the arena -- this might wait for any readers to finish
+  _mi_arenas_page_unabandon(page, theap);
+  _mi_theap_page_reclaim(theap, page);
+  mi_theap_stat_counter_increase(theap, pages_reclaim_on_free, 1);
+  return true;
+}
+
+
+// We freed a block in an abandoned page (that was not owned). Try to collect
+static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept
+{
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_assert_internal(mt_free != NULL);
+  // we own the page now, and it is safe to collect the thread atomic free list
+  if (page->block_size <= MI_SMALL_SIZE_MAX) {
+    // use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list
+    // (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted).
+    //  however, if the page became completely free, the used count is guaranteed to be 0.)
+    mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used.
+    _mi_page_free_collect_partly(page, mt_free);    
+  }
+  else {
+    // for larger blocks we use the regular collect 
+    _mi_page_free_collect(page,false /* no force */);
+    mt_free = NULL; // expected page->xthread_free value after collection
+  }
+  const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free);
+  #if MI_DEBUG > 1
+  if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); }
+  if (mi_page_is_full(page))      { mi_assert(mi_page_is_mostly_used(page)); }
+  #endif
+
+  // try to: 1. free it, 2. reclaim it, or 3. reabandon it to be mapped
+  if (mi_abandoned_page_try_free(page)) return;
+  if (page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE && reclaim_on_free >= 0) {  // early test for better codegen
+    if (mi_abandoned_page_try_reclaim(page, reclaim_on_free)) return;
+  }
+  if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
+  
+  // otherwise unown the page again
+  mi_abandoned_page_unown_from_free(page, mt_free);
+}
+
+
+// ------------------------------------------------------
+// Usable size
+// ------------------------------------------------------
+
+// Bytes available in a block
+static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept {
+  const mi_block_t* block = _mi_page_ptr_unalign(page, p);
+  const bool is_guarded = mi_block_ptr_is_guarded(block,p);
+  const size_t size = mi_page_usable_size_of(page, block, is_guarded);
+  const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;
+  mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
+  const size_t aligned_size = (size - adjust);  
+  return aligned_size;
+}
+
+static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept {
+  if mi_unlikely(page==NULL) return 0;
+  if mi_likely(!mi_page_has_interior_pointers(page)) {
+    const mi_block_t* block = (const mi_block_t*)p;
+    return mi_page_usable_size_of(page, block, false /* is guarded */);
+  }
+  else {
+    // split out to separate routine for improved code generation
+    return mi_page_usable_aligned_size_of(page, p);
+  }
+}
+
+mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept {
+  const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size");
+  return _mi_usable_size(p,page);
+}
+
+
+// ------------------------------------------------------
+// Free variants
+// ------------------------------------------------------
+
+void mi_free_size(void* p, size_t size) mi_attr_noexcept {
+  MI_UNUSED_RELEASE(size);
+  #if MI_DEBUG
+  const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size");  
+  const size_t available = _mi_usable_size(p,page);
+  mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ );
+  #endif
+  mi_free(p);
+}
+
+void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {
+  MI_UNUSED_RELEASE(alignment);
+  mi_assert(((uintptr_t)p % alignment) == 0);
+  mi_free_size(p,size);
+}
+
+void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
+  MI_UNUSED_RELEASE(alignment);
+  mi_assert(((uintptr_t)p % alignment) == 0);
+  mi_free(p);
+}
+
+
+// ------------------------------------------------------
+// Check for double free in secure and debug mode
+// This is somewhat expensive so only enabled for secure mode 4
+// ------------------------------------------------------
+
+#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))
+// linear check if the free list contains a specific element
+static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) {
+  while (list != NULL) {
+    if (elem==list) return true;
+    list = mi_block_next(page, list);
+  }
+  return false;
+}
+
+static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) {
+  // The decoded value is in the same page (or NULL).
+  // Walk the free lists to verify positively if it is already freed
+  if (mi_list_contains(page, page->free, block) ||
+      mi_list_contains(page, page->local_free, block) ||
+      mi_list_contains(page, mi_page_thread_free(page), block))
+  {
+    _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page));
+    return true;
+  }
+  return false;
+}
+
+#define mi_track_page(page,access)  { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); }
+
+static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
+  bool is_double_free = false;
+  mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field
+  if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 &&  // quick check: aligned pointer?
+      (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL?
+  {
+    // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free?
+    // (continue in separate function to improve code generation)
+    is_double_free = mi_check_is_double_freex(page, block);
+  }
+  return is_double_free;
+}
+#else
+static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
+  MI_UNUSED(page);
+  MI_UNUSED(block);
+  return false;
+}
+#endif
+
+
+// ---------------------------------------------------------------------------
+// Check for theap block overflow by setting up padding at the end of the block
+// ---------------------------------------------------------------------------
+
+#if MI_PADDING // && !MI_TRACK_ENABLED
+static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) {
+  *bsize = mi_page_usable_block_size(page);
+  const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);
+  mi_track_mem_defined(padding,sizeof(mi_padding_t));
+  *delta = padding->delta;
+  uint32_t canary = padding->canary;
+  uintptr_t keys[2];
+  keys[0] = page->keys[0];
+  keys[1] = page->keys[1];
+  bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize);
+  mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
+  return ok;
+}
+
+// Return the exact usable size of a block.
+static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
+  if (is_guarded) {
+    const size_t bsize = mi_page_block_size(page);
+    return (bsize - _mi_os_page_size());
+  }
+  else {
+    size_t bsize;
+    size_t delta;
+    bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
+    mi_assert_internal(ok); mi_assert_internal(delta <= bsize);
+    return (ok ? bsize - delta : 0);
+  }
+}
+
+// When a non-thread-local block is freed, it becomes part of the thread delayed free
+// list that is freed later by the owning theap. If the exact usable size is too small to
+// contain the pointer for the delayed list, then shrink the padding (by decreasing delta)
+// so it will later not trigger an overflow error in `mi_free_block`.
+void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
+  size_t bsize;
+  size_t delta;
+  bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
+  mi_assert_internal(ok);
+  if (!ok || (bsize - delta) >= min_size) return;  // usually already enough space
+  mi_assert_internal(bsize >= min_size);
+  if (bsize < min_size) return;  // should never happen
+  size_t new_delta = (bsize - min_size);
+  mi_assert_internal(new_delta < bsize);
+  mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize);
+  mi_track_mem_defined(padding,sizeof(mi_padding_t));
+  padding->delta = (uint32_t)new_delta;
+  mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
+}
+#else
+static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
+  MI_UNUSED(is_guarded); MI_UNUSED(block);
+  return mi_page_usable_block_size(page);
+}
+
+void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
+  MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(min_size);
+}
+#endif
+
+#if MI_PADDING && MI_PADDING_CHECK
+
+static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) {
+  size_t bsize;
+  size_t delta;
+  bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
+  *size = *wrong = bsize;
+  if (!ok) return false;
+  mi_assert_internal(bsize >= delta);
+  *size = bsize - delta;
+  if (!mi_page_is_huge(page)) {
+    uint8_t* fill = (uint8_t*)block + bsize - delta;
+    const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes
+    mi_track_mem_defined(fill, maxpad);
+    for (size_t i = 0; i < maxpad; i++) {
+      if (fill[i] != MI_DEBUG_PADDING) {
+        *wrong = bsize - delta + i;
+        ok = false;
+        break;
+      }
+    }
+    mi_track_mem_noaccess(fill, maxpad);
+  }
+  return ok;
+}
+
+static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
+  size_t size;
+  size_t wrong;
+  if (!mi_verify_padding(page,block,&size,&wrong)) {
+    _mi_error_message(EFAULT, "buffer overflow in theap block %p of size %zu: write after %zu bytes\n", block, size, wrong );
+  }
+}
+
+#else
+
+static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
+  MI_UNUSED(page);
+  MI_UNUSED(block);
+}
+
+#endif
+
+// only maintain stats for smaller objects if requested
+#if (MI_STAT>0)
+static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
+  MI_UNUSED(block);
+  mi_theap_t* const theap = _mi_theap_default();
+  if (!mi_theap_is_initialized(theap)) return; // (for now) skip statistics if free'd after thread_done was called (usually a thread cleanup call by the OS)
+
+  const size_t bsize = mi_page_usable_block_size(page);
+  // #if (MI_STAT>1)
+  // const size_t usize = mi_page_usable_size_of(page, block);
+  // mi_theap_stat_decrease(theap, malloc_requested, usize);
+  // #endif
+  if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
+    mi_theap_stat_decrease(theap, malloc_normal, bsize);
+    #if (MI_STAT > 1)
+    mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1);
+    #endif
+  }
+  else {
+    const size_t bpsize = mi_page_block_size(page);  // match stat in page.c:mi_huge_page_alloc
+    mi_theap_stat_decrease(theap, malloc_huge, bpsize);
+  }
+}
+#else
+void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
+  MI_UNUSED(page); MI_UNUSED(block);
+}
+#endif
+
+
+// Remove guard page when building with MI_GUARDED
+#if MI_GUARDED
+static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {
+  MI_UNUSED(p);
+  mi_assert_internal(mi_block_ptr_is_guarded(block, p));
+  mi_assert_internal(mi_page_has_interior_pointers(page));
+  mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));
+  mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);
+
+  const size_t bsize = mi_page_block_size(page);
+  const size_t psize = _mi_os_page_size();
+  mi_assert_internal(bsize > psize);
+  mi_assert_internal(!page->memid.is_pinned);
+  void* gpage = (uint8_t*)block + bsize - psize;
+  mi_assert_internal(_mi_is_aligned(gpage, psize));
+  _mi_os_unprotect(gpage, psize);
+}
+#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/heap.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/heap.c
new file mode 100644
index 00000000..f0a01652
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/heap.c
@@ -0,0 +1,273 @@
+/*----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"  // _mi_theap_default
+
+
+/* -----------------------------------------------------------
+  Heap's
+----------------------------------------------------------- */
+
+mi_theap_t* mi_heap_theap(mi_heap_t* heap) {
+  return _mi_heap_theap(heap);
+}
+
+void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node) {
+  if (heap==NULL) { heap = mi_heap_main(); }
+  heap->numa_node = (numa_node < 0 ? -1 : numa_node % _mi_os_numa_node_count());
+}
+
+void mi_heap_stats_merge_to_subproc(mi_heap_t* heap) {
+  if (heap==NULL) { heap = mi_heap_main(); }
+  _mi_stats_merge_into(&heap->subproc->stats, &heap->stats);
+}
+
+void mi_heap_stats_merge_to_main(mi_heap_t* heap) {
+  if (heap==NULL) return;
+  _mi_stats_merge_into(&mi_heap_main()->stats, &heap->stats);
+}
+
+static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_heap)
+{
+  mi_heap_t* heap = (mi_heap_t*)const_heap;
+  mi_assert_internal(heap!=NULL);
+
+  if (_mi_is_heap_main(heap)) {
+    // this can be called if the (main) thread is not yet initialized (as no allocation happened)
+    // but `theap_main_init_get()` will call `mi_thread_init()`
+    mi_theap_t* const theap = _mi_theap_main_safe();
+    mi_assert_internal(theap!=NULL && _mi_is_heap_main(_mi_theap_heap(theap)));
+    return theap;
+  }
+
+  // otherwise initialize the theap for this heap
+  // get the thread local
+  mi_assert_internal(heap->theap != 0);
+  if (heap->theap==0) {  // paranoia
+    _mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap);
+    return NULL;
+  }
+  mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);
+
+  // create a fresh theap?
+  if (theap==NULL) {
+    // set first an invalid value to ensure the thread local storage is allocated
+    if (!_mi_thread_local_set(heap->theap, (mi_theap_t*)1)) {
+      _mi_error_message(EFAULT, "unable to allocate memory for thread local storage\n");
+      return NULL;
+    }    
+    // then allocate the theap
+    theap = _mi_theap_create(heap, _mi_theap_default_safe()->tld);
+    _mi_thread_local_set(heap->theap, theap);  // Cannot fail now as it was set before. Always set so the local is valid or NULL (and not 1)
+    if (theap==NULL) {
+      _mi_error_message(EFAULT, "unable to allocate memory for a thread local heap\n");
+      return NULL;
+    }    
+  }
+  return theap;
+}
+
+
+// get the theap for a heap without initializing (and return NULL in that case)
+mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) {
+  if (heap==NULL || _mi_is_heap_main(heap)) {
+    return _mi_theap_main_safe(); 
+  }
+  else {
+    return (mi_theap_t*)_mi_thread_local_get(heap->theap);
+  }
+}
+
+// get (and possibly create) the theap belonging to a heap
+mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap)
+{
+  mi_theap_t* theap = _mi_heap_theap_peek(heap);
+  if mi_unlikely(theap==NULL) {
+    theap = mi_heap_init_theap(heap);
+    if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; }  // this will return NULL from page.c:_mi_malloc_generic
+  }
+  _mi_theap_cached_set(theap);
+  return theap;
+}
+
+
+mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {
+  // always allocate heap data in the (subprocess) main heap
+  mi_heap_t* const heap_main = mi_heap_main();
+  // todo: allocate heap data in the exclusive arena ?
+  mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) );
+  if (heap==NULL) return NULL;
+
+  // reserve a thread local slot for this heap (see also issue #1230)
+  const mi_thread_local_t theap_slot = _mi_thread_local_create();
+  if (theap_slot == 0) {
+    _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n");
+    mi_free(heap);
+    return NULL;
+  }
+
+  // init fields
+  heap->theap = theap_slot;
+  heap->subproc = heap_main->subproc;
+  heap->heap_seq = mi_atomic_increment_relaxed(&heap_main->subproc->heap_total_count);
+  heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id);
+  heap->numa_node = -1; // no initial affinity
+  mi_stats_header_init(&heap->stats);
+  mi_lock_init(&heap->theaps_lock);
+  mi_lock_init(&heap->os_abandoned_pages_lock);
+  mi_lock_init(&heap->arena_pages_lock);
+
+  // push onto the subproc heaps
+  mi_lock(&heap->subproc->heaps_lock) {
+    mi_heap_t* head = heap->subproc->heaps;
+    heap->prev = NULL;
+    heap->next = head;
+    if (head!=NULL) { head->prev = heap;  }
+    heap->subproc->heaps = heap;
+  }
+  mi_atomic_increment_relaxed(&heap_main->subproc->heap_count);
+  mi_subproc_stat_increase(heap_main->subproc, heaps, 1);
+  return heap;
+}
+
+mi_heap_t* mi_heap_new(void) {
+  return mi_heap_new_in_arena(0);
+}
+
+// free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency)
+static void mi_heap_free_theaps(mi_heap_t* heap) {
+  // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`), 
+  // and we need to ensure we free theaps atomically.
+  // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock 
+  // potential concurrent thread termination which tries to remove the theap from our theaps list.
+  bool all_freed;
+  do {
+    all_freed = true;
+    mi_theap_t* theap = NULL;
+    mi_lock(&heap->theaps_lock) { 
+      theap = heap->theaps; 
+      while(theap != NULL) {
+        mi_theap_t* next = theap->hnext;
+        if (!_mi_theap_free(theap, false /* dont re-acquire the heap->theaps_lock */, true /* acquire the tld->theaps_lock though */ )) {
+          all_freed = false;
+        }
+        theap = next;
+      }      
+    }
+    if (!all_freed) { 
+      mi_heap_stat_counter_increase(heap,heaps_delete_wait,1); 
+      _mi_prim_thread_yield();
+    }
+    else { 
+      mi_assert_internal(heap->theaps==NULL); 
+    }               
+  }
+  while(!all_freed);
+}
+
+// free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed)
+static void mi_heap_free(mi_heap_t* heap) {
+  mi_assert_internal(heap!=NULL && !_mi_is_heap_main(heap));
+
+  // free all arena pages infos
+  mi_lock(&heap->arena_pages_lock) {
+    for (size_t i = 0; i < MI_MAX_ARENAS; i++) {
+      mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
+      if (arena_pages!=NULL) {
+        mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);
+        mi_free(arena_pages);
+      }
+    }
+  }
+
+  // remove the heap from the subproc
+  mi_heap_stats_merge_to_main(heap);
+  mi_atomic_decrement_relaxed(&heap->subproc->heap_count);
+  mi_subproc_stat_decrease(heap->subproc, heaps, 1);
+  mi_lock(&heap->subproc->heaps_lock) {
+    if (heap->next!=NULL) { heap->next->prev = heap->prev; }
+    if (heap->prev!=NULL) { heap->prev->next = heap->next; }
+                     else { heap->subproc->heaps = heap->next; }
+  }
+
+  _mi_thread_local_free(heap->theap);
+  mi_lock_done(&heap->theaps_lock);
+  mi_lock_done(&heap->os_abandoned_pages_lock);
+  mi_lock_done(&heap->arena_pages_lock);
+  mi_free(heap);
+}
+
+void mi_heap_delete(mi_heap_t* heap) {
+  if (heap==NULL) return;
+  if (_mi_is_heap_main(heap)) {
+    _mi_warning_message("cannot delete the main heap\n");
+    return;
+  }
+  mi_heap_free_theaps(heap);
+  _mi_heap_move_pages(heap, mi_heap_main());
+  mi_heap_free(heap);
+}
+
+void _mi_heap_force_destroy(mi_heap_t* heap) {
+  if (heap==NULL) return;
+  mi_heap_free_theaps(heap);
+  _mi_heap_destroy_pages(heap);
+  if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); }  // todo: release locks of the main heap?
+}
+
+void mi_heap_destroy(mi_heap_t* heap) {
+  if (heap==NULL) return;
+  if (_mi_is_heap_main(heap)) {
+    _mi_warning_message("cannot destroy the main heap\n");
+    return;
+  }
+  _mi_heap_force_destroy(heap);
+}
+
+mi_heap_t* mi_heap_of(const void* p) {
+  mi_page_t* page = _mi_safe_ptr_page(p);
+  if (page==NULL) return NULL;
+  return mi_page_heap(page);
+}
+
+bool mi_any_heap_contains(const void* p) {
+  return (mi_heap_of(p)!=NULL);
+}
+
+bool mi_heap_contains(const mi_heap_t* heap, const void* p) {
+  if (heap==NULL) { heap = mi_heap_main(); }
+  return (heap==mi_heap_of(p));
+}
+
+// deprecated
+bool mi_check_owned(const void* p) {
+  return mi_any_heap_contains(p);
+}
+
+// unsafe heap utilization function for DragonFly (see issue #1258)
+// If the page of pointer `p` belongs to `heap` (or `heap==NULL`) and has less than `perc_threshold` used blocks in its used area return `true`.
+// This function is unsafe in general as it assumes we are the only thread accessing the page of `p`.
+bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept {
+  if (p==NULL) return false;
+  const mi_page_t* const page = _mi_safe_ptr_page(p);   // Get the page containing this pointer
+  if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false;
+  // If the page is the head of the queue, it is currently being used for 
+  // allocations; we skip it to avoid immediate thrashing.
+  if (page->prev == NULL)  return false;
+
+  // match heap?
+  const mi_heap_t* const page_heap = mi_page_heap(page);
+  if (page_heap==NULL) return false;
+  if (heap!=NULL && page_heap!=heap) return false;
+    
+  // check utilization
+  if (page->capacity==0)   return false;
+  if (perc_threshold>=100) return true;
+  return (perc_threshold >= ((100UL*page->used) / page->capacity));
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/init.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/init.c
new file mode 100644
index 00000000..1fabd891
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/init.c
@@ -0,0 +1,1199 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"
+
+#include <string.h>  // memcpy, memset
+#include <stdlib.h>  // atexit
+
+#define MI_MEMID_INIT(kind)   {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ }
+#define MI_MEMID_STATIC       MI_MEMID_INIT(MI_MEM_STATIC)
+
+// Empty page used to initialize the small free pages array
+const mi_page_t _mi_page_empty = {
+  MI_ATOMIC_VAR_INIT(0),  // xthread_id
+  NULL,                   // free
+  0,                      // used
+  0,                      // capacity
+  0,                      // reserved capacity
+  0,                      // retire_expire
+  false,                  // is_zero
+  NULL,                   // local_free
+  MI_ATOMIC_VAR_INIT(0),  // xthread_free
+  0,                      // block_size
+  NULL,                   // page_start
+  #if (MI_PADDING || MI_ENCODE_FREELIST)
+  { 0, 0 },               // keys
+  #endif
+  NULL,                   // theap
+  NULL,                   // heap
+  NULL, NULL,             // next, prev
+  MI_ARENA_SLICE_SIZE,    // page_committed
+  MI_MEMID_STATIC         // memid
+};
+
+#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)
+
+#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8)
+#define MI_SMALL_PAGES_EMPTY  { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
+#elif (MI_PADDING>0)
+#define MI_SMALL_PAGES_EMPTY  { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
+#else
+#define MI_SMALL_PAGES_EMPTY  { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }
+#endif
+
+
+// Empty page queues for every bin
+#define QNULL(sz)  { NULL, NULL, 0, (sz)*sizeof(uintptr_t) }
+#define MI_PAGE_QUEUES_EMPTY \
+  { QNULL(1), \
+    QNULL(     1), QNULL(     2), QNULL(     3), QNULL(     4), QNULL(     5), QNULL(     6), QNULL(     7), QNULL(     8), /* 8 */ \
+    QNULL(    10), QNULL(    12), QNULL(    14), QNULL(    16), QNULL(    20), QNULL(    24), QNULL(    28), QNULL(    32), /* 16 */ \
+    QNULL(    40), QNULL(    48), QNULL(    56), QNULL(    64), QNULL(    80), QNULL(    96), QNULL(   112), QNULL(   128), /* 24 */ \
+    QNULL(   160), QNULL(   192), QNULL(   224), QNULL(   256), QNULL(   320), QNULL(   384), QNULL(   448), QNULL(   512), /* 32 */ \
+    QNULL(   640), QNULL(   768), QNULL(   896), QNULL(  1024), QNULL(  1280), QNULL(  1536), QNULL(  1792), QNULL(  2048), /* 40 */ \
+    QNULL(  2560), QNULL(  3072), QNULL(  3584), QNULL(  4096), QNULL(  5120), QNULL(  6144), QNULL(  7168), QNULL(  8192), /* 48 */ \
+    QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \
+    QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \
+    QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \
+    QNULL(MI_LARGE_MAX_OBJ_WSIZE + 1  /* 655360, Huge queue */), \
+    QNULL(MI_LARGE_MAX_OBJ_WSIZE + 2) /* Full queue */ }
+
+#define MI_STAT_COUNT_NULL()  {0,0,0}
+
+// Empty statistics
+#define MI_STAT_COUNT(stat)     {0,0,0},
+#define MI_STAT_COUNTER(stat)   {0},
+
+#define MI_STATS_NULL  \
+  MI_STAT_FIELDS() \
+  \
+  { MI_INIT4(MI_STAT_COUNT_NULL) }, \
+  { { 0 }, { 0 }, { 0 }, { 0 } }, \
+  \
+  { MI_INIT74(MI_STAT_COUNT_NULL) }, \
+  { MI_INIT74(MI_STAT_COUNT_NULL) }, \
+  { MI_INIT5(MI_STAT_COUNT_NULL) }
+
+// --------------------------------------------------------
+// Statically allocate an empty theap as the initial
+// thread local value for the default theap,
+// and statically allocate the backing theap for the main
+// thread so it can function without doing any allocation
+// itself (as accessing a thread local for the first time
+// may lead to allocation itself on some platforms)
+// --------------------------------------------------------
+
+static mi_decl_cache_align mi_subproc_t subproc_main
+#if __cplusplus
+  = { };     // empty initializer to prevent running the constructor (with msvc)
+#else
+  = { 0 };   // C zero initialize
+#endif
+
+static mi_subproc_t* subprocs = &subproc_main;
+static mi_lock_t     subprocs_lock;
+
+static mi_decl_cache_align mi_tld_t tld_empty = {
+  0,                      // thread_id
+  0,                      // thread_seq
+  0,                      // default numa node
+  &subproc_main,          // subproc
+  NULL,                   // theaps list
+  MI_LOCK_INITIALIZER,    // theaps lock
+  false,                  // recurse
+  false,                  // is_in_threadpool
+  MI_MEMID_STATIC         // memid
+};
+
+mi_decl_cache_align const mi_theap_t _mi_theap_empty = {
+  &tld_empty,             // tld
+  MI_ATOMIC_VAR_INIT(NULL), // heap
+  MI_ATOMIC_VAR_INIT(1),  // refcount
+  0,                      // heartbeat
+  0,                      // cookie
+  { {0}, {0}, 0, true },  // random
+  0,                      // page count
+  MI_BIN_FULL, 0,         // page retired min/max
+  0,                      // pages_full_size
+  0, 0,                   // generic count
+  NULL, NULL,             // tnext, tprev
+  NULL, NULL,             // hnext, hprev
+  0,                      // full page retain
+  false,                  // allow reclaim
+  true,                   // allow abandon
+  #if MI_GUARDED
+  0, 0, 0, 1,             // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)
+  #endif
+  MI_SMALL_PAGES_EMPTY,
+  MI_PAGE_QUEUES_EMPTY,
+  MI_MEMID_STATIC,
+  { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL },      // stats
+};
+
+mi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = {
+  &tld_empty,             // tld
+  MI_ATOMIC_VAR_INIT(NULL), // heap
+  MI_ATOMIC_VAR_INIT(1),  // refcount
+  0,                      // heartbeat
+  0,                      // cookie
+  { {0}, {0}, 0, true },  // random
+  0,                      // page count
+  MI_BIN_FULL, 0,         // page retired min/max
+  0,                      // pages_full_size
+  0, 0,                   // generic count
+  NULL, NULL,             // tnext, tprev
+  NULL, NULL,             // hnext, hprev
+  0,                      // full page retain
+  false,                  // allow reclaim
+  true,                   // allow abandon
+  #if MI_GUARDED
+  0, 0, 0, 1,             // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)
+  #endif
+  MI_SMALL_PAGES_EMPTY,
+  MI_PAGE_QUEUES_EMPTY,
+  MI_MEMID_STATIC,
+  { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL },      // stats
+};
+
+// Heap for the main thread
+
+extern mi_decl_hidden mi_decl_cache_align mi_theap_t theap_main;
+extern mi_decl_hidden mi_decl_cache_align mi_heap_t  heap_main;
+
+static mi_decl_cache_align mi_tld_t tld_main = {
+  0,                      // thread_id
+  0,                      // thread_seq
+  0,                      // numa node
+  &subproc_main,          // subproc
+  &theap_main,            // theaps list
+  MI_LOCK_INITIALIZER,    // theaps lock
+  false,                  // recurse
+  false,                  // is_in_threadpool
+  MI_MEMID_STATIC         // memid
+};
+
+mi_decl_cache_align mi_theap_t theap_main = {
+  &tld_main,              // thread local data
+  MI_ATOMIC_VAR_INIT(&heap_main), // main heap
+  MI_ATOMIC_VAR_INIT(1),  // refcount
+  0,                      // heartbeat
+  0,                      // initial cookie
+  { {0x846ca68b}, {0}, 0, true },  // random
+  0,                      // page count
+  MI_BIN_FULL, 0,         // page retired min/max
+  0,                      // pages_full_size
+  0, 0,                   // generic count
+  NULL, NULL,             // tnext, tprev
+  NULL, NULL,             // hnext, hprev
+  2,                      // full page retain
+  true,                   // allow page reclaim
+  true,                   // allow page abandon
+  #if MI_GUARDED
+  0, 0, 0, 0,
+  #endif
+  MI_SMALL_PAGES_EMPTY,
+  MI_PAGE_QUEUES_EMPTY,
+  MI_MEMID_STATIC,
+  { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL },      // stats
+};
+
+mi_decl_cache_align mi_heap_t heap_main
+#if __cplusplus
+  = { };     // empty initializer to prevent running the constructor (with msvc)
+#else
+  = { 0 };   // C zero initialize
+#endif
+
+// the theap belonging to the main heap
+mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main = NULL;
+
+mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
+  mi_threadid_t tid = _mi_prim_thread_id();
+  mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits
+  return tid;
+}
+
+#if MI_TLS_MODEL_THREAD_LOCAL
+// the thread-local main theap for allocation
+mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty;
+// the last used non-main theap
+mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty;
+#endif
+
+bool _mi_process_is_initialized = false;  // set to `true` in `mi_process_init`.
+
+mi_stats_t _mi_stats_main = { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL };
+
+#undef MI_STAT_COUNT
+#undef MI_STAT_COUNTER
+
+
+#if MI_GUARDED
+mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {
+  theap->guarded_sample_rate  = sample_rate;
+  theap->guarded_sample_count = sample_rate;  // count down samples
+  if (theap->guarded_sample_rate > 1) {
+    if (seed == 0) {
+      seed = _mi_theap_random_next(theap);
+    }
+    theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1;  // start at random count between 1 and `sample_rate`
+  }
+}
+
+mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {
+  theap->guarded_size_min = min;
+  theap->guarded_size_max = (min > max ? min : max);
+}
+
+void _mi_theap_guarded_init(mi_theap_t* theap) {
+  mi_theap_guarded_set_sample_rate(theap,
+    (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX),
+    (size_t)mi_option_get(mi_option_guarded_sample_seed));
+  mi_theap_guarded_set_size_bound(theap,
+    (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX),
+    (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) );
+}
+#else
+mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {
+  MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed);
+}
+
+mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {
+  MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max);
+}
+void _mi_theap_guarded_init(mi_theap_t* theap) {
+  MI_UNUSED(theap);
+}
+#endif
+
+/* -----------------------------------------------------------
+  Initialization
+  Note: on some platforms lock_init or just a thread local access
+  can cause allocation and induce recursion during initialization.
+----------------------------------------------------------- */
+
+
+// Initialize main subproc
+static void mi_subproc_main_init(void) {
+  if (subproc_main.memid.memkind != MI_MEM_STATIC) {
+    subproc_main.memid = _mi_memid_create(MI_MEM_STATIC);
+    subproc_main.heaps = &heap_main;
+    subproc_main.heap_total_count = 1;
+    subproc_main.heap_count = 1;
+    mi_atomic_store_ptr_release(mi_heap_t, &subproc_main.heap_main, &heap_main);
+    __mi_stat_increase_mt(&subproc_main.stats.heaps, 1);
+    mi_stats_header_init(&subproc_main.stats);
+    mi_lock_init(&subproc_main.arena_reserve_lock);
+    mi_lock_init(&subproc_main.heaps_lock);
+    mi_lock_init(&subprocs_lock);
+    mi_lock_init(&tld_empty.theaps_lock); 
+  }
+}
+
+// Initialize main tld
+static void mi_tld_main_init(void) {
+  if (tld_main.thread_id == 0) {
+    tld_main.thread_id = _mi_prim_thread_id();
+    mi_lock_init(&tld_main.theaps_lock);
+  }
+}
+
+void _mi_theap_options_init(mi_theap_t* theap) {
+  theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0);
+  theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0);
+  theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);
+}
+
+// Initialization of the (statically allocated) main theap, and the main tld and subproc.
+static void mi_theap_main_init(void) {
+  if mi_unlikely(theap_main.memid.memkind != MI_MEM_STATIC) {
+    // theap
+    theap_main.memid = _mi_memid_create(MI_MEM_STATIC);
+    #if defined(__APPLE__) || defined(_WIN32) && !defined(MI_SHARED_LIB)
+      _mi_random_init_weak(&theap_main.random);    // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)
+    #else
+      _mi_random_init(&theap_main.random);
+    #endif
+    theap_main.cookie  = _mi_theap_random_next(&theap_main);
+    _mi_theap_options_init(&theap_main);
+    _mi_theap_guarded_init(&theap_main);
+  }
+}
+
+// Initialize main heap
+static void mi_heap_main_init(void) {
+  if mi_unlikely(heap_main.subproc == NULL) {
+    heap_main.subproc = &subproc_main;
+    heap_main.theaps = &theap_main;
+
+    mi_theap_main_init();
+    mi_subproc_main_init();
+    mi_tld_main_init();
+
+    mi_lock_init(&heap_main.theaps_lock);
+    mi_lock_init(&heap_main.os_abandoned_pages_lock);
+    mi_lock_init(&heap_main.arena_pages_lock);
+  }
+}
+
+
+/* -----------------------------------------------------------
+  Thread local data
+----------------------------------------------------------- */
+
+// Allocate fresh tld
+static mi_tld_t* mi_tld_alloc(void) {
+  if (_mi_is_main_thread()) {
+    mi_atomic_increment_relaxed(&tld_main.subproc->thread_count);
+    return &tld_main;
+  }
+  else {
+    // allocate tld meta-data
+    // note: we need to be careful to not access the tld from `_mi_meta_zalloc`
+    // (and in turn from `_mi_arena_alloc_aligned` and `_mi_os_alloc_aligned`).
+    mi_memid_t memid;
+    mi_tld_t* tld = (mi_tld_t*)_mi_meta_zalloc(sizeof(mi_tld_t), &memid);
+    if (tld==NULL) {
+      _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n");
+      return NULL;
+    }
+    tld->memid = memid;
+    tld->theaps = NULL;
+    mi_lock_init(&tld->theaps_lock);
+    tld->subproc = &subproc_main;
+    tld->numa_node = _mi_os_numa_node();
+    tld->thread_id = _mi_prim_thread_id();
+    tld->thread_seq = mi_atomic_increment_relaxed(&tld->subproc->thread_total_count);
+    tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool();
+    mi_atomic_increment_relaxed(&tld->subproc->thread_count);
+    return tld;
+  }
+}
+
+#define MI_TLD_INVALID  ((mi_tld_t*)1)
+
+mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) {
+  mi_lock_done(&tld->theaps_lock);
+  if (tld != NULL && tld != MI_TLD_INVALID) {
+    mi_atomic_decrement_relaxed(&tld->subproc->thread_count);
+    _mi_meta_free(tld, sizeof(mi_tld_t), tld->memid);
+  }
+  #if 0
+  // do not read/write to `thread_tld` on older macOS <= 14 as that will re-initialize the thread local storage
+  // (since we are calling this during pthread shutdown)
+  // (and this could happen on other systems as well, so let's never do it)
+  thread_tld = MI_TLD_INVALID;
+  #endif
+}
+
+// return the thread local heap ensuring it is initialized (and not `NULL` or `&_mi_theap_empty`);
+mi_theap_t* _mi_theap_default_safe(void) {
+  mi_theap_t* theap = _mi_theap_default();
+  if mi_likely(mi_theap_is_initialized(theap)) return theap;
+  mi_thread_init();
+  mi_assert_internal(mi_theap_is_initialized(_mi_theap_default()));
+  return _mi_theap_default();
+}
+
+// return the main theap ensuring it is initialized. 
+mi_theap_t* _mi_theap_main_safe(void) {
+  mi_theap_t* theap = __mi_theap_main;
+  if mi_unlikely(theap==NULL) {  // if thread_init or default_set was never called
+    mi_thread_init();            // sets the default slot to the main theap
+    theap = _mi_theap_default();
+    mi_assert_internal(theap!=NULL);
+    mi_assert_internal(_mi_is_theap_main(theap));
+    if (_mi_is_theap_main(theap)) {
+      __mi_theap_main = theap;
+    }
+  }
+  mi_assert_internal(theap!=NULL && _mi_is_theap_main(theap));    
+  return theap;
+}
+
+
+mi_subproc_t* _mi_subproc_main(void) {
+  return &subproc_main;
+}
+
+mi_subproc_t* _mi_subproc(void) {
+  // should work without doing initialization (as it may be called from `_mi_tld -> mi_tld_alloc ... -> os_alloc -> _mi_subproc()`
+  // todo: this will still fail on OS systems where the first access to a thread-local causes allocation.
+  //       on such systems we can check for this with the _mi_prim_get_default_theap as those are protected (by being
+  //       stored in a TLS slot for example)
+  mi_theap_t* theap = _mi_theap_default();
+  if (theap == NULL) {
+    return _mi_subproc_main();
+  }
+  else {
+    return theap->tld->subproc;  // avoid using thread local storage (`thread_tld`)
+  }
+}
+
+mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) {
+  mi_heap_t* heap = mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);
+  if mi_likely(heap!=NULL) {
+    return heap;
+  }
+  else {
+    mi_heap_main_init();
+    mi_assert_internal(mi_atomic_load_relaxed(&subproc->heap_main) != NULL);
+    return mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);
+  }
+}
+
+mi_heap_t* mi_heap_main(void) {
+  return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init
+}
+
+bool _mi_is_heap_main(const mi_heap_t* heap) {
+  mi_assert_internal(heap!=NULL);
+  return (_mi_subproc_heap_main(heap->subproc) == heap);
+}
+
+bool _mi_is_theap_main(const mi_theap_t* theap) {
+  return (mi_theap_is_initialized(theap) && _mi_is_heap_main(_mi_theap_heap(theap)));
+}
+
+/* -----------------------------------------------------------
+  Sub process
+----------------------------------------------------------- */
+
+
+mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {
+  return (mi_subproc_t*)(subproc_id._mi_subproc_id);
+}
+
+mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) {
+  mi_subproc_id_t id = { subproc };
+  return id;
+}
+
+mi_subproc_id_t mi_subproc_main(void) {
+  return _mi_subproc_to_id(_mi_subproc_main());
+}
+
+mi_subproc_id_t mi_subproc_current(void) {
+  return _mi_subproc_to_id(_mi_subproc());
+}
+
+mi_subproc_id_t mi_subproc_new(void) {
+  static _Atomic(size_t) subproc_total_count;
+  mi_memid_t memid;
+  mi_subproc_t* subproc = (mi_subproc_t*)_mi_meta_zalloc(sizeof(mi_subproc_t),&memid);
+  if (subproc == NULL) return _mi_subproc_to_id(NULL);
+  subproc->memid = memid;
+  subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count) + 1;
+  mi_stats_header_init(&subproc->stats);
+  mi_lock_init(&subproc->arena_reserve_lock);
+  mi_lock_init(&subproc->heaps_lock);
+  mi_lock(&subprocs_lock) {
+    // push on subproc list
+    subproc->next = subprocs;
+    if (subprocs!=NULL) { subprocs->prev = subproc; }
+    subprocs = subproc;
+  }
+  return _mi_subproc_to_id(subproc);
+}
+
+// destroy all subproc resources including arena's, heap's etc.
+static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock)
+{
+  if (subproc==NULL) return;
+
+  // remove from the subproc list
+  mi_lock_maybe(&subprocs_lock, acquire_subprocs_lock) {
+    if (subproc->next!=NULL) { subproc->next->prev = subproc->prev;  }
+    if (subproc->prev!=NULL) { subproc->prev->next = subproc->next;  }
+                        else { mi_assert_internal(subprocs==subproc);  subprocs = subproc->next; }
+  }
+
+  // destroy all subproc heaps
+  mi_lock(&subproc->heaps_lock) {
+    mi_heap_t* heap = subproc->heaps;
+    while (heap != NULL) {
+      mi_heap_t* next = heap->next;
+      if (heap!=subproc->heap_main) { mi_heap_destroy(heap); }
+      heap = next;
+    }
+    mi_assert_internal(subproc->heaps == subproc->heap_main);
+    _mi_heap_force_destroy(subproc->heap_main);  // no warning if destroying the main heap
+  }
+
+  // remove associated arenas
+  _mi_arenas_unsafe_destroy_all(subproc);
+
+  // merge stats back into the main subproc?
+  if (subproc!=&subproc_main) {
+    _mi_stats_merge_into(&subproc_main.stats, &subproc->stats);
+  }
+
+  // safe to release
+  // todo: should we refcount subprocesses?
+  mi_lock_done(&subproc->arena_reserve_lock);
+  mi_lock_done(&subproc->heaps_lock);
+  if (subproc!=&subproc_main) {
+    _mi_meta_free(subproc, sizeof(mi_subproc_t), subproc->memid);
+  }
+  else {
+    // for the main subproc, also release the global page map
+    _mi_page_map_unsafe_destroy(&subproc_main);
+  }
+}
+
+void mi_subproc_destroy(mi_subproc_id_t subproc_id) {
+  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+  if (subproc==NULL || subproc==&subproc_main) return;
+  mi_subproc_unsafe_destroy(subproc, true /* take lock */);
+}
+
+static void mi_subprocs_unsafe_destroy_all(void) {
+  mi_lock(&subprocs_lock) {
+    mi_subproc_t* subproc = subprocs;
+    while (subproc!=NULL) {
+      mi_subproc_t* next = subproc->next;
+      if (subproc!=&subproc_main) {
+        mi_subproc_unsafe_destroy(subproc, false /* take subprocs lock */);
+      }
+      subproc = next;
+    }
+  }  
+  mi_subproc_unsafe_destroy(&subproc_main, true /* take subprocs lock */);
+}
+
+
+void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {
+  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+  mi_tld_t* const tld = _mi_theap_default_safe()->tld;
+  mi_assert(tld->subproc== &subproc_main);
+  if (tld->subproc != &subproc_main) {
+    _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (id: %p)\n", subproc);
+    return;
+  }
+  tld->subproc = subproc;
+  tld->thread_seq = mi_atomic_increment_relaxed(&subproc->thread_total_count);
+  mi_atomic_decrement_relaxed(&subproc_main.thread_count);
+  mi_atomic_increment_relaxed(&subproc->thread_count);
+}
+
+
+bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) {
+  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+  if (subproc==NULL) return false;
+  bool ok = true;
+  mi_lock(&subproc->heaps_lock) {
+    for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) {
+      ok = (*visitor)(heap, arg);
+    }
+  }
+  return ok;
+}
+
+
+/* -----------------------------------------------------------
+  Allocate theap data
+----------------------------------------------------------- */
+
+// Initialize the thread local default theap, called from `mi_thread_init`
+static mi_theap_t* _mi_thread_init_theap_default(void) {
+  mi_theap_t* theap = _mi_theap_default();
+  if (mi_theap_is_initialized(theap)) return theap;
+  if (_mi_is_main_thread()) {
+    mi_heap_main_init();
+    theap = &theap_main;
+  }
+  else {
+    // allocates tld data
+    // note: we cannot access thread-locals yet as that can cause (recursive) allocation
+    // (on macOS <= 14 for example where the loader allocates thread-local data on demand).
+    mi_tld_t* tld = mi_tld_alloc();
+    if (tld==NULL) return NULL;  // things are very wrong if this fails (out of memory)
+    // allocate and initialize the theap for the main heap
+    theap = _mi_theap_create(mi_heap_main(), tld);
+  }
+  // associate the theap with this thread
+  // (this is safe, on macOS for example, the theap is set in a dedicated TLS slot and thus does not cause recursive allocation)
+  _mi_theap_default_set(theap);
+  return theap;
+}
+
+
+// Free the thread local theaps
+static void mi_thread_theaps_done(mi_tld_t* tld)
+{
+  // reset the thread local theaps
+  _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty);
+  _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty);
+  __mi_theap_main = NULL;
+  
+  // abandon the pages of all theaps in this thread
+  mi_lock(&tld->theaps_lock) {
+    mi_theap_t* theap = tld->theaps;
+    while (theap != NULL) {
+      mi_theap_t* next = theap->tnext; 
+      // never destroy theaps; if a dll is linked statically with mimalloc,
+      // there may still be delete/free calls after the mi_fls_done is called. Issue #207
+      _mi_theap_collect_abandon(theap);
+      mi_assert_internal(theap->page_count==0);
+      theap = next;
+    }
+  }
+
+  // free the theaps of this thread.
+  // This can run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically.
+  // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock 
+  // potential concurrent `mi_heap_free_theaps` which tries to remove the theap from our theaps list.
+  bool all_freed;
+  do {
+    all_freed = true;
+    mi_lock(&tld->theaps_lock) {
+      mi_theap_t* theap = tld->theaps;
+      while (theap != NULL) {
+        mi_theap_t* next = theap->tnext;
+        mi_assert_internal(theap->page_count==0);
+        if (!_mi_theap_free(theap, true /* acquire heap->theaps_lock */, false /* dont re-acquire the tld->theaps_lock*/ )) {
+          all_freed = false;
+        }
+        theap = next;
+      }
+    }
+    if (!all_freed) { 
+      mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1); 
+      _mi_prim_thread_yield(); 
+    }
+    else { 
+      mi_assert_internal(tld->theaps==NULL); 
+    }
+  } while (!all_freed);
+
+  mi_assert(_mi_theap_default()==(mi_theap_t*)&_mi_theap_empty); // careful to not re-initialize the default theap during theap_delete
+  mi_assert(!mi_theap_is_initialized(_mi_theap_default()));
+}
+
+
+
+// --------------------------------------------------------
+// Try to run `mi_thread_done()` automatically so any memory
+// owned by the thread but not yet released can be abandoned
+// and re-owned by another thread.
+//
+// 1. windows dynamic library:
+//     call from DllMain on DLL_THREAD_DETACH
+// 2. windows static library:
+//     use special linker section to call a destructor when the thread is done
+// 3. unix, pthreads:
+//     use a pthread key to call a destructor when a pthread is done
+//
+// In the last two cases we also need to call `mi_process_init`
+// to set up the thread local keys.
+// --------------------------------------------------------
+
+// Set up handlers so `mi_thread_done` is called automatically
+static void mi_process_setup_auto_thread_done(void) {
+  mi_atomic_do_once {
+    _mi_prim_thread_init_auto_done();
+    _mi_theap_default_set(&theap_main);
+  }
+}
+
+
+bool _mi_is_main_thread(void) {
+  return (tld_main.thread_id==0 || tld_main.thread_id == _mi_thread_id());
+}
+
+
+// Initialize thread
+void mi_thread_init(void) mi_attr_noexcept
+{
+  // ensure our process has started already
+  mi_process_init();
+  // if the theap_default is already set we have already initialized
+  if (_mi_thread_is_initialized()) return;
+
+  // initialize the default theap
+  if (_mi_thread_init_theap_default() == NULL) return; // out-of-memory on tld/theap allocation
+
+  mi_heap_stat_increase(mi_heap_main(), threads, 1);
+  // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
+}
+
+void mi_thread_done(void) mi_attr_noexcept {
+  _mi_thread_done(NULL);
+}
+
+void _mi_thread_done(mi_theap_t* _theap_main)
+{
+  // NULL can be passed on some platforms
+  if (_theap_main==NULL) {
+    _theap_main = __mi_theap_main;  // don't call `mi_theap_main_safe` as that re-initializes the thread
+    if (_theap_main==NULL) {        // can happen if `mi_theap_main_safe` is never called; but then the default is main
+      _theap_main = _mi_theap_default();
+      mi_assert_internal(_theap_main==NULL || _mi_is_theap_main(_theap_main));
+    }
+  }
+
+  // prevent re-entrancy through theap_done/theap_set_default_direct (issue #699)
+  if (!mi_theap_is_initialized(_theap_main)) {
+    return;
+  }
+
+  // release dynamic thread_local's
+  _mi_thread_locals_thread_done();
+
+  // note: we store the tld as we should avoid reading `thread_tld` at this point (to avoid reinitializing the thread local storage)
+  mi_tld_t* const tld = _theap_main->tld;
+
+  // adjust stats
+  mi_heap_stat_decrease(_mi_subproc_heap_main(tld->subproc), threads, 1);  // todo: or `_theap_main->heap`?
+
+  // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local theaps...
+  if (tld->thread_id != _mi_prim_thread_id()) return;
+
+  // delete the thread local theaps
+  mi_thread_theaps_done(tld);
+
+  // free thread local data
+  mi_tld_free(tld);
+}
+
+
+mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) {
+  return (mi_theap_t*)&_mi_theap_empty;
+}
+
+#if MI_TLS_MODEL_DYNAMIC_WIN32
+
+// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed)
+// See <https://en.wikipedia.org/wiki/Win32_Thread_Information_Block> for the offsets.
+#if MI_SIZE_SIZE==4
+#define MI_TLS_DIRECT_FIRST             (0x0E10 / MI_SIZE_SIZE)
+#else
+#define MI_TLS_DIRECT_FIRST             (0x1480 / MI_SIZE_SIZE)
+#endif
+#define MI_TLS_DIRECT_SLOTS             (64)
+#define MI_TLS_EXPANSION_SLOTS          (1024)
+
+#if !MI_WIN_DIRECT_TLS
+#define MI_TLS_INITIAL_SLOT             MI_TLS_EXPANSION_SLOT
+#define MI_TLS_INITIAL_EXPANSION_SLOT   (MI_TLS_EXPANSION_SLOTS-1)
+#else
+// with only direct entries, use the "arbitrary user data" field 
+// and assume it is NULL (see also <http://www.nynaeve.net/?p=98>)
+#define MI_TLS_INITIAL_SLOT             (5)
+#define MI_TLS_INITIAL_EXPANSION_SLOT   (0)
+#endif
+
+// we initially use the last of the expansion slots as the default NULL.
+// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc (which is quite unlikely)
+mi_decl_hidden mi_decl_cache_align size_t _mi_theap_default_slot = MI_TLS_INITIAL_SLOT;
+mi_decl_hidden size_t _mi_theap_default_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT;
+mi_decl_hidden size_t _mi_theap_cached_slot            = MI_TLS_INITIAL_SLOT;
+mi_decl_hidden size_t _mi_theap_cached_expansion_slot  = MI_TLS_INITIAL_EXPANSION_SLOT;
+
+static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES;
+static DWORD mi_tls_raw_index_cached  = TLS_OUT_OF_INDEXES;
+
+static bool mi_win_tls_slot_alloc(size_t* slot, size_t* extended, DWORD* raw_index) {
+  const DWORD index = TlsAlloc();
+  *raw_index = index;
+  if (index==TLS_OUT_OF_INDEXES) {
+    *extended = 0;
+    *slot = 0;
+    return false;
+  }
+  else if (index<MI_TLS_DIRECT_SLOTS) {
+    *extended = 0;
+    *slot = index + MI_TLS_DIRECT_FIRST;
+    return true;
+  }
+  #if !MI_WIN_DIRECT_TLS
+  else if (index < MI_TLS_DIRECT_SLOTS + MI_TLS_EXPANSION_SLOTS - 1) { // check maximum number of expansion slots - 1 (as we use the last one as the default)    
+    *extended = index - MI_TLS_DIRECT_SLOTS;
+    *slot = MI_TLS_EXPANSION_SLOT;
+    return true;
+  }
+  #endif
+  else {
+    // to high an index for us
+    _mi_error_message(EFAULT, "returned tls index was too high (%u)\n", index);
+    TlsFree(index);
+    *raw_index = TLS_OUT_OF_INDEXES; 
+    *extended = 0;
+    *slot = 0;
+    return false;
+  }
+}
+
+static void mi_win_tls_slot_free(DWORD* raw_index) {
+  if (*raw_index != TLS_OUT_OF_INDEXES) {
+    TlsFree(*raw_index);
+    *raw_index = TLS_OUT_OF_INDEXES;
+  }
+}
+
+static void mi_tls_slots_init(void) {
+  mi_atomic_do_once {
+    bool ok = mi_win_tls_slot_alloc(&_mi_theap_default_slot, &_mi_theap_default_expansion_slot, &mi_tls_raw_index_default);
+    if (ok) {
+      ok = mi_win_tls_slot_alloc(&_mi_theap_cached_slot, &_mi_theap_cached_expansion_slot, &mi_tls_raw_index_cached);
+    }
+    if (!ok) {
+      _mi_error_message(EFAULT, "unable to allocate fast TLS user slot (0x%zx)\n", _mi_theap_cached_slot);
+    }
+  }
+}
+
+static void mi_tls_slots_done(void) {
+  mi_win_tls_slot_free(&mi_tls_raw_index_default);
+  mi_win_tls_slot_free(&mi_tls_raw_index_cached );
+}
+
+static void mi_win_tls_slot_set(size_t slot, size_t extended_slot, void* value) {
+  mi_assert_internal((slot >= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT);
+  if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) {
+    mi_prim_tls_slot_set(slot, value);
+  }
+  else {
+    mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS);
+    TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value);  // use TlsSetValue to initialize the TlsExpansion array if needed
+  }
+}
+
+#elif MI_TLS_MODEL_DYNAMIC_PTHREADS
+
+// only for pthreads for now
+mi_decl_hidden pthread_key_t _mi_theap_default_key = 0;
+mi_decl_hidden pthread_key_t _mi_theap_cached_key = 0;
+
+// create a non-zero pthread key
+static int mi_pthread_key_create( pthread_key_t* pkey ) {
+  pthread_key_t key;
+  int err = pthread_key_create(&key, NULL);
+  if (err!=0) return err;
+  if (key==0) {
+    // if we get a zero key, create another one as we use 0 for an invalid key
+    pthread_key_t key2;
+    err = pthread_key_create(&key2, NULL);
+    pthread_key_delete(key);  // delete the old key
+    if (err!=0) return err;
+    key = key2;
+  }
+  mi_assert_internal(key!=0);    
+  *pkey = key;
+  return 0;
+}
+
+static void mi_tls_slots_init(void) {
+  mi_atomic_do_once {
+    int err = mi_pthread_key_create(&_mi_theap_default_key);
+    if (err==0) {
+      err = mi_pthread_key_create(&_mi_theap_cached_key);
+    }
+    if (err!=0) {
+      _mi_error_message(EFAULT, "unable to allocate pthread keys (error %d)\n", err);
+    }
+  }
+}
+
+static void mi_tls_slots_done(void) {
+  if (_mi_theap_default_key != 0) {
+    pthread_key_delete(_mi_theap_default_key);
+    _mi_theap_default_key = 0;
+  }
+  if (_mi_theap_cached_key != 0) {
+    pthread_key_delete(_mi_theap_cached_key);
+    _mi_theap_cached_key = 0;
+  }
+}
+
+#else
+
+static void mi_tls_slots_init(void) {
+  // nothing
+}
+
+static void mi_tls_slots_done(void) {
+  // nothing
+}
+
+#endif
+
+void _mi_theap_cached_set(mi_theap_t* theap) {
+  mi_theap_t* prev = _mi_theap_cached();
+  if (prev==theap) return;
+  // set
+  mi_tls_slots_init();
+  #if MI_TLS_MODEL_THREAD_LOCAL
+    __mi_theap_cached = theap;
+  #elif MI_TLS_MODEL_FIXED_SLOT
+    mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_CACHED, theap);
+  #elif MI_TLS_MODEL_DYNAMIC_WIN32
+    mi_win_tls_slot_set(_mi_theap_cached_slot, _mi_theap_cached_expansion_slot, theap);
+  #elif MI_TLS_MODEL_DYNAMIC_PTHREADS
+    if (_mi_theap_cached_key!=0) pthread_setspecific(_mi_theap_cached_key, theap);
+  #endif
+  // update refcounts (so cached theap memory keeps available until no longer cached)
+  _mi_theap_incref(theap);
+  _mi_theap_decref(prev);    
+}
+
+void _mi_theap_default_set(mi_theap_t* theap)  {
+  mi_theap_t* const theap_old = _mi_theap_default();
+  mi_assert_internal(theap != NULL);
+  mi_assert_internal(theap->tld->thread_id==0 || theap->tld->thread_id==_mi_thread_id());
+  mi_tls_slots_init();
+  #if MI_TLS_MODEL_THREAD_LOCAL
+    __mi_theap_default = theap;
+  #elif MI_TLS_MODEL_FIXED_SLOT
+    mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_DEFAULT, theap);
+  #elif MI_TLS_MODEL_DYNAMIC_WIN32
+    mi_win_tls_slot_set(_mi_theap_default_slot, _mi_theap_default_expansion_slot, theap);
+  #elif MI_TLS_MODEL_DYNAMIC_PTHREADS
+    if (_mi_theap_default_key!=0) pthread_setspecific(_mi_theap_default_key, theap);
+  #endif
+
+  // set theap main if needed
+  if (mi_theap_is_initialized(theap)) {
+    // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again)
+    _mi_prim_thread_associate_default_theap(theap);
+    if (_mi_is_heap_main(_mi_theap_heap(theap))) {
+      __mi_theap_main = theap;
+    }
+  }
+
+  // ensure either the default slot contains the main theap, or __mi_theap_main is initialized 
+  if (mi_theap_is_initialized(theap_old) && _mi_is_heap_main(_mi_theap_heap(theap_old))) {
+    __mi_theap_main = theap_old;
+  }
+}
+
+void mi_thread_set_in_threadpool(void) mi_attr_noexcept {
+  mi_theap_t* theap = _mi_theap_default_safe();
+  theap->tld->is_in_threadpool = true;
+}
+
+// --------------------------------------------------------
+// Run functions on process init/done, and thread init/done
+// --------------------------------------------------------
+static bool os_preloading = true;    // true until this module is initialized
+
+// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.
+bool mi_decl_noinline _mi_preloading(void) {
+  return os_preloading;
+}
+
+// Returns true if mimalloc was redirected
+mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept {
+  return _mi_is_redirected();
+}
+
+// Called once by the process loader from `src/prim/prim.c`
+void _mi_auto_process_init(void) {
+  // mi_heap_main_init();
+  // #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)
+  // volatile mi_theap_t* dummy = __mi_theap_default; // access TLS to allocate it before setting tls_initialized to true;
+  // if (dummy == NULL) return;                       // use dummy or otherwise the access may get optimized away (issue #697)
+  // #endif
+
+  os_preloading = false;
+  mi_assert_internal(_mi_is_main_thread());
+
+  mi_process_init();
+  mi_process_setup_auto_thread_done();
+  _mi_thread_locals_init();
+  _mi_options_post_init();  // now we can print to stderr
+  if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n");
+
+  // show message from the redirector (if present)
+  const char* msg = NULL;
+  _mi_allocator_init(&msg);
+  if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {
+    _mi_fputs(NULL,NULL,NULL,msg);
+  }
+
+  // reseed random
+  _mi_random_reinit_if_weak(&theap_main.random);
+}
+
+// CPU features
+mi_decl_cache_align size_t _mi_cpu_movsb_max = 0;  // for size <= max, rep movsb is fast
+mi_decl_cache_align size_t _mi_cpu_stosb_max = 0;  // for size <= max, rep stosb is fast
+mi_decl_cache_align bool _mi_cpu_has_popcnt = false;
+
+#if (MI_ARCH_X64 || MI_ARCH_X86)
+#if defined(__GNUC__)
+// #include <cpuid.h>
+static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
+  // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx)
+  // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect).
+  uint32_t eax, ebx, ecx, edx;
+  __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : );
+  regs4[0] = eax;
+  regs4[1] = ebx;
+  regs4[2] = ecx;
+  regs4[3] = edx;
+  return true;
+}
+
+#elif defined(_MSC_VER)
+static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
+  __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel);
+  return true;
+}
+#else
+static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
+  MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel);
+  return false;
+}
+#endif
+
+static void mi_detect_cpu_features(void) {
+  // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))
+  // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?)
+  // FSRS for fast short rep stosb
+  bool amd = false;
+  bool fsrm = false;
+  // bool erms = false;
+  bool fsrs = false;
+  uint32_t cpu_info[4];
+  if (mi_cpuid(cpu_info, 0, 0)) {
+    amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD)
+  }
+  if (mi_cpuid(cpu_info, 7, 0)) {
+    fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
+    // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
+  }
+  if (mi_cpuid(cpu_info, 7, 1)) {
+    fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=1:_Extended_Features>
+  }
+  if (mi_cpuid(cpu_info, 1, 0)) {
+    _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see <https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits>
+  }
+
+  if (fsrm) {
+    _mi_cpu_movsb_max = 127;
+  }
+  if (fsrs || (amd && fsrm)) {  // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803
+    _mi_cpu_stosb_max = 127;
+  }
+}
+
+#else
+static void mi_detect_cpu_features(void) {
+  #if MI_ARCH_ARM64
+  _mi_cpu_has_popcnt = true;
+  #endif
+}
+#endif
+
+
+// Initialize the process; called by thread_init or the process loader
+static void mi_process_init_once(void) mi_attr_noexcept {
+  _mi_process_is_initialized = true;  
+  _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id());
+
+  mi_detect_cpu_features();
+  _mi_options_init();
+  _mi_stats_init();
+  _mi_os_init();
+  // the following can potentially allocate (on freeBSD for pthread keys)
+  // todo: do 2-phase so we can use stats at first, then later init the keys?
+  mi_heap_main_init(); // before page_map_init so stats are working
+  _mi_page_map_init(); // todo: this could fail.. should we abort in that case?
+  mi_thread_init();
+  _mi_process_is_initialized = true;
+
+  #if defined(_WIN32) && defined(MI_WIN_USE_FLS)
+  // On windows, when building as a static lib the FLS cleanup happens to early for the main thread.
+  // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup
+  // will not call _mi_thread_done on the (still executing) main thread. See issue #508.
+  _mi_prim_thread_associate_default_theap(NULL);
+  #endif
+
+  // mi_stats_reset();  // only call stat reset *after* thread init (or the theap tld == NULL)
+  mi_track_init();
+  if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
+    size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024);
+    int reserve_at  = (int)mi_option_get_clamp(mi_option_reserve_huge_os_pages_at, -1, INT_MAX);
+    if (reserve_at != -1) {
+      mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500);
+    } else {
+      mi_reserve_huge_os_pages_interleave(pages, 0, pages*500);
+    }
+  }
+  if (mi_option_is_enabled(mi_option_reserve_os_memory)) {
+    long ksize = mi_option_get(mi_option_reserve_os_memory);
+    if (ksize > 0) {
+      mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true);
+    }
+  }
+}
+
+// Initialize the process; called by thread_init or the process loader
+void mi_process_init(void) mi_attr_noexcept {
+  // #if _MSC_VER < 1920
+	// mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main
+	// #endif
+  mi_atomic_do_once {
+    mi_process_init_once();
+  }
+}
+
+// Called when the process is done (cdecl as it is used with `at_exit` on some platforms)
+void mi_cdecl mi_process_done(void) mi_attr_noexcept {
+  // only shutdown if we were initialized
+  if (!_mi_process_is_initialized) return;
+  // ensure we are called once
+  static bool process_done = false;
+  if (process_done) return;
+  process_done = true;
+
+  // free dynamic thread locals (if used at all)
+  _mi_thread_locals_done();
+
+  // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread
+  _mi_prim_thread_done_auto_done();
+
+  #ifndef MI_SKIP_COLLECT_ON_EXIT
+    #if (MI_DEBUG || !defined(MI_SHARED_LIB))
+    // free all memory if possible on process exit. This is not needed for a stand-alone process
+    // but should be done if mimalloc is statically linked into another shared library which
+    // is repeatedly loaded/unloaded, see issue #281.
+    mi_theap_collect(_mi_theap_default(), true /* force */);
+    #endif
+  #endif
+
+  // done with tracking tools
+  mi_track_done()
+
+  // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free
+  // since after process_done there might still be other code running that calls `free` (like at_exit routines,
+  // or C-runtime termination code.
+  if (mi_option_is_enabled(mi_option_destroy_on_exit)) {
+    mi_subprocs_unsafe_destroy_all(); // destroys all subprocs, arenas, and the page_map!    
+  }
+  else {
+    mi_heap_stats_merge_to_subproc(mi_heap_main());
+  }
+  
+  // careful now to no longer access any allocator functionality 
+  if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {
+    mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL);
+  }
+  mi_lock_done(&subprocs_lock);
+  mi_tls_slots_done();
+  _mi_allocator_done();
+  _mi_verbose_message("process done: 0x%zx\n", tld_main.thread_id);
+  os_preloading = true; // don't call the C runtime anymore
+}
+
+void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept {
+  if (_mi_option_get_fast(mi_option_destroy_on_exit)>1) return;
+  mi_process_done();
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/libc.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/libc.c
new file mode 100644
index 00000000..c74003a1
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/libc.c
@@ -0,0 +1,477 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// --------------------------------------------------------
+// This module defines various std libc functions to reduce
+// the dependency on libc, and also prevent errors caused
+// by some libc implementations when called before `main`
+// executes (due to malloc redirection)
+// --------------------------------------------------------
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"      // mi_prim_getenv
+
+char _mi_toupper(char c) {
+  if (c >= 'a' && c <= 'z') return (c - 'a' + 'A');
+                       else return c;
+}
+
+int _mi_strnicmp(const char* s, const char* t, size_t n) {
+  mi_assert_internal(s!=NULL && t!=NULL);
+  if (n == 0) return 0;
+  for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {
+    if (_mi_toupper(*s) != _mi_toupper(*t)) break;
+  }
+  return (n == 0 ? 0 : *s - *t);
+}
+
+bool _mi_streq(const char* s, const char* t) {
+  if (s==NULL && t==NULL) return true;
+  if (s==NULL || t==NULL) return false;
+  for (; *s != 0 && *t != 0; s++, t++) {
+    if (*s != *t) break;
+  }
+  return (*s == *t);
+}
+
+void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
+  if (dest==NULL || src==NULL || dest_size == 0) return;
+  // copy until end of src, or when dest is (almost) full
+  while (*src != 0 && dest_size > 1) {
+    *dest++ = *src++;
+    dest_size--;
+  }
+  // always zero terminate
+  *dest = 0;
+}
+
+void _mi_strlcat(char* dest, const char* src, size_t dest_size) {
+  if (dest==NULL || src==NULL || dest_size == 0) return;
+  // find end of string in the dest buffer
+  while (*dest != 0 && dest_size > 1) {
+    dest++;
+    dest_size--;
+  }
+  // and catenate
+  _mi_strlcpy(dest, src, dest_size);
+}
+
+size_t _mi_strnlen(const char* s, size_t max_len) {
+  if (s==NULL) return 0;
+  size_t len = 0;
+  while(s[len] != 0 && len < max_len) { len++; }
+  return len;
+}
+
+size_t _mi_strlen(const char* s) {
+  return _mi_strnlen(s,PTRDIFF_MAX);
+}
+
+char* _mi_strnstr(char* s, size_t max_len, const char* pat) {
+  if (s==NULL) return NULL;
+  if (pat==NULL) return s;
+  const size_t m = _mi_strnlen(s, max_len);
+  const size_t n = _mi_strlen(pat);  
+  for (size_t start = 0; start + n <= m; start++) {
+    size_t i = 0;
+    while (i<n && pat[i]==s[start+i]) {
+      i++;
+    }
+    if (i==n) return &s[start];
+  }
+  return NULL;
+}
+
+#ifdef MI_NO_GETENV
+bool _mi_getenv(const char* name, char* result, size_t result_size) {
+  MI_UNUSED(name);
+  MI_UNUSED(result);
+  MI_UNUSED(result_size);
+  return false;
+}
+#else
+bool _mi_getenv(const char* name, char* result, size_t result_size) {
+  if (name==NULL || result == NULL || result_size < 64) return false;
+  return _mi_prim_getenv(name,result,result_size);
+}
+#endif
+
+
+// --------------------------------------------------------
+// Define our own primitives for doing an action once
+// --------------------------------------------------------
+
+// Returns `true` only on the first invocation, signifying we can execute an action once.
+// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
+// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
+bool _mi_atomic_once_enter(mi_atomic_once_t* once) {
+  const uintptr_t once_tid = mi_atomic_load_acquire(&once->tid);
+  if mi_likely(once_tid == 1) {
+    return false; // already executed
+  }
+  const mi_threadid_t current_tid = _mi_thread_id();
+  if (once_tid == current_tid) {
+    return false; // recursive invocation; we need this for process_init for example
+  }  
+
+  mi_lock_acquire(&once->lock);
+  uintptr_t expected = 0;
+  if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) {  // could use atomic_load/store as well
+    return true;  // should execute and release
+  } 
+  else {
+    mi_lock_release(&once->lock);
+    return false; // already another thread entered and released
+  }
+}
+
+void _mi_atomic_once_release(mi_atomic_once_t* once) {
+  if (mi_atomic_load_acquire(&once->tid)>1) {  // paranoia
+    mi_atomic_store_release(&once->tid,1);     // done executing
+    mi_lock_release(&once->lock);
+  }  
+}
+
+
+// --------------------------------------------------------
+// Define our own limited `_mi_vsnprintf` and `_mi_snprintf`
+// This is mostly to avoid calling these when libc is not yet
+// initialized (and to reduce dependencies)
+//
+// format:      d i, p x u, s
+// prec:        z l ll L
+// width:       10
+// align-left:  -
+// fill:        0
+// plus:        +
+// --------------------------------------------------------
+
+static void mi_outc(char c, char** out, char* end) {
+  char* p = *out;
+  if (p >= end) return;
+  *p = c;
+  *out = p + 1;
+}
+
+static void mi_outs(const char* s, char** out, char* end) {
+  if (s == NULL) return;
+  char* p = *out;
+  while (*s != 0 && p < end) {
+    *p++ = *s++;
+  }
+  *out = p;
+}
+
+static void mi_out_fill(char fill, size_t len, char** out, char* end) {
+  char* p = *out;
+  for (size_t i = 0; i < len && p < end; i++) {
+    *p++ = fill;
+  }
+  *out = p;
+}
+
+static void mi_out_alignright(char fill, char* start, size_t len, size_t extra, char* end) {
+  if (len == 0 || extra == 0) return;
+  if (start + len + extra >= end) return;
+  // move `len` characters to the right (in reverse since it can overlap)
+  for (size_t i = 1; i <= len; i++) {
+    start[len + extra - i] = start[len - i];
+  }
+  // and fill the start
+  for (size_t i = 0; i < extra; i++) {
+    start[i] = fill;
+  }
+}
+
+
+static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* end)
+{
+  if (x == 0 || base == 0 || base > 16) {
+    if (prefix != 0) { mi_outc(prefix, out, end); }
+    mi_outc('0',out,end);
+  }
+  else {
+    // output digits in reverse
+    char* start = *out;
+    while (x > 0) {
+      char digit = (char)(x % base);
+      mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end);
+      x = x / base;
+    }
+    if (prefix != 0) {
+      mi_outc(prefix, out, end);
+    }
+    size_t len = *out - start;
+    // and reverse in-place
+    for (size_t i = 0; i < (len / 2); i++) {
+      char c = start[len - i - 1];
+      start[len - i - 1] = start[i];
+      start[i] = c;
+    }
+  }
+}
+
+
+#define MI_NEXTC()  c = *in; if (c==0) break; in++;
+
+int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
+  if (buf == NULL || bufsize == 0 || fmt == NULL) return 0;
+  buf[bufsize - 1] = 0;
+  char* const end = buf + (bufsize - 1);
+  const char* in = fmt;
+  char* out = buf;
+  while (true) {
+    if (out >= end) break;
+    char c;
+    MI_NEXTC();
+    if (c != '%') {
+      if (c == '\\') {
+        MI_NEXTC();
+        switch (c) {
+        case 'e': mi_outc('\x1B', &out, end); break;
+        case 't': mi_outc('\t', &out, end); break;
+        case 'n': mi_outc('\n', &out, end); break;
+        case 'r': mi_outc('\r', &out, end); break;
+        case '\\': mi_outc('\\', &out, end); break;
+        default: /* ignore */ break;
+        }
+      }
+      else if ((c >= ' ' && c <= '~') || c=='\n' || c=='\r' || c=='\t' || c=='\x1b') { // output visible ascii or standard control only
+        mi_outc(c, &out, end);
+      }
+    }
+    else {
+      MI_NEXTC();
+      char   fill = ' ';
+      size_t width = 0;
+      char   numtype = 'd';
+      char   numplus = 0;
+      bool   alignright = true;
+      if (c == '+' || c == ' ') { numplus = c; MI_NEXTC(); }
+      if (c == '-') { alignright = false; MI_NEXTC(); }
+      if (c == '0') { fill = '0'; MI_NEXTC(); }
+      if (c >= '1' && c <= '9') {
+        width = (c - '0'); MI_NEXTC();
+        while (c >= '0' && c <= '9') {
+          width = (10 * width) + (c - '0'); MI_NEXTC();
+        }
+        if (c == 0) break;  // extra check due to while
+      }
+      if (c == 'z' || c == 't' || c == 'L') { numtype = c; MI_NEXTC(); }
+      else if (c == 'l') {
+        numtype = c; MI_NEXTC();
+        if (c == 'l') { numtype = 'L'; MI_NEXTC(); }
+      }
+
+      char* start = out;
+      if (c == '%') {
+        mi_outc('%', &out, end);
+      }
+      else if (c == 's') {
+        // string
+        const char* s = va_arg(args, const char*);
+        mi_outs(s, &out, end);
+      }
+      else if (c == 'p' || c == 'x' || c == 'u') {
+        // unsigned
+        uintmax_t x = 0;
+        if (c == 'x' || c == 'u') {
+          if (numtype == 'z')       x = va_arg(args, size_t);
+          else if (numtype == 't')  x = va_arg(args, uintptr_t); // unsigned ptrdiff_t
+          else if (numtype == 'L')  x = va_arg(args, unsigned long long);
+          else if (numtype == 'l')  x = va_arg(args, unsigned long);
+                               else x = va_arg(args, unsigned int);
+        }
+        else if (c == 'p') {
+          void* const p = va_arg(args, void*);
+          x = (uintptr_t)p;
+          mi_outs("0x", &out, end);
+          start = out;
+          width = (width >= 2 ? width - 2 : 0);
+        }
+        if (width == 0 && (c == 'x' || c == 'p')) {
+          if (c == 'p')   { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); }
+          if (width == 0) { width = 2; }
+          fill = '0';
+        }
+        mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end);
+      }
+      else if (c == 'i' || c == 'd') {
+        // signed
+        intmax_t x = 0;
+        if (numtype == 'z')       x = va_arg(args, intptr_t );
+        else if (numtype == 't')  x = va_arg(args, ptrdiff_t);
+        else if (numtype == 'L')  x = va_arg(args, long long);
+        else if (numtype == 'l')  x = va_arg(args, long);
+                             else x = va_arg(args, int);
+        char pre = 0;
+        if (x < 0) {
+          pre = '-';
+          if (x > INTMAX_MIN) { x = -x; }
+        }
+        else if (numplus != 0) {
+          pre = numplus;
+        }
+        mi_out_num((uintmax_t)x, 10, pre, &out, end);
+      }
+      else if (c >= ' ' && c <= '~') {
+        // unknown format
+        mi_outc('%', &out, end);
+        mi_outc(c, &out, end);
+      }
+
+      // fill & align
+      mi_assert_internal(out <= end);
+      mi_assert_internal(out >= start);
+      const size_t len = out - start;
+      if (len < width) {
+        mi_out_fill(fill, width - len, &out, end);
+        if (alignright && out <= end) {
+          mi_out_alignright(fill, start, len, width - len, end);
+        }
+      }
+    }
+  }
+  mi_assert_internal(out <= end);
+  *out = 0;
+  return (int)(out - buf);
+}
+
+int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) {
+  va_list args;
+  va_start(args, fmt);
+  const int written = _mi_vsnprintf(buf, buflen, fmt, args);
+  va_end(args);
+  return written;
+}
+
+
+
+// --------------------------------------------------------
+// generic trailing and leading zero count, and popcount
+// --------------------------------------------------------
+
+#if !MI_HAS_FAST_BITSCAN
+
+static size_t mi_ctz_generic32(uint32_t x) {
+  // de Bruijn multiplication, see <http://keithandkatie.com/keith/papers/debruijn.html>
+  static const uint8_t debruijn[32] = {
+    0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+    31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
+  };
+  if (x==0) return 32;
+  return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27];
+}
+
+static size_t mi_clz_generic32(uint32_t x) {
+  // de Bruijn multiplication, see <http://keithandkatie.com/keith/papers/debruijn.html>
+  static const uint8_t debruijn[32] = {
+    31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1,
+    23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0
+  };
+  if (x==0) return 32;
+  x |= x >> 1;
+  x |= x >> 2;
+  x |= x >> 4;
+  x |= x >> 8;
+  x |= x >> 16;
+  return debruijn[(uint32_t)(x * (uint32_t)(0x07C4ACDDU)) >> 27];
+}
+
+size_t _mi_ctz_generic(size_t x) {
+  if (x==0) return MI_SIZE_BITS;
+  #if (MI_SIZE_BITS <= 32)
+    return mi_ctz_generic32((uint32_t)x);
+  #else
+    const uint32_t lo = (uint32_t)x;
+    if (lo != 0) {
+      return mi_ctz_generic32(lo);
+    }
+    else {
+      return (32 + mi_ctz_generic32((uint32_t)(x>>32)));
+    }
+  #endif
+}
+
+size_t _mi_clz_generic(size_t x) {
+  if (x==0) return MI_SIZE_BITS;
+  #if (MI_SIZE_BITS <= 32)
+    return mi_clz_generic32((uint32_t)x);
+  #else
+    const uint32_t hi = (uint32_t)(x>>32);
+    if (hi != 0) {
+      return mi_clz_generic32(hi);
+    }
+    else {
+      return 32 + mi_clz_generic32((uint32_t)x);
+    }
+  #endif
+}
+
+#endif // bit scan
+
+
+#if MI_SIZE_SIZE == 4
+#define mi_mask_even_bits32      (0x55555555)
+#define mi_mask_even_pairs32     (0x33333333)
+#define mi_mask_even_nibbles32   (0x0F0F0F0F)
+
+// sum of all the bytes in `x` if it is guaranteed that the sum < 256!
+static size_t mi_byte_sum32(uint32_t x) {
+  // perform `x * 0x01010101`: the highest byte contains the sum of all bytes.
+  x += (x << 8);
+  x += (x << 16);
+  return (size_t)(x >> 24);
+}
+
+static size_t mi_popcount_generic32(uint32_t x) {
+  // first count each 2-bit group `a`, where: a==0b00 -> 00, a==0b01 -> 01, a==0b10 -> 01, a==0b11 -> 10
+  // in other words, `a - (a>>1)`; to do this in parallel, we need to mask to prevent spilling a bit pair
+  // into the lower bit-pair:
+  x = x - ((x >> 1) & mi_mask_even_bits32);
+  // add the 2-bit pair results
+  x = (x & mi_mask_even_pairs32) + ((x >> 2) & mi_mask_even_pairs32);
+  // add the 4-bit nibble results
+  x = (x + (x >> 4)) & mi_mask_even_nibbles32;
+  // each byte now has a count of its bits, we can sum them now:
+  return mi_byte_sum32(x);
+}
+
+mi_decl_noinline size_t _mi_popcount_generic(size_t x) {
+  if (x<=1) return x;
+  if (~x==0) return MI_SIZE_BITS;
+  return mi_popcount_generic32(x);
+}
+
+#else
+#define mi_mask_even_bits64      (0x5555555555555555)
+#define mi_mask_even_pairs64     (0x3333333333333333)
+#define mi_mask_even_nibbles64   (0x0F0F0F0F0F0F0F0F)
+
+// sum of all the bytes in `x` if it is guaranteed that the sum < 256!
+static size_t mi_byte_sum64(uint64_t x) {
+  x += (x << 8);
+  x += (x << 16);
+  x += (x << 32);
+  return (size_t)(x >> 56);
+}
+
+static size_t mi_popcount_generic64(uint64_t x) {
+  x = x - ((x >> 1) & mi_mask_even_bits64);
+  x = (x & mi_mask_even_pairs64) + ((x >> 2) & mi_mask_even_pairs64);
+  x = (x + (x >> 4)) & mi_mask_even_nibbles64;
+  return mi_byte_sum64(x);
+}
+
+mi_decl_noinline size_t _mi_popcount_generic(size_t x) {
+  if (x<=1) return x;
+  if (~x==0) return MI_SIZE_BITS;
+  return mi_popcount_generic64(x);
+}
+#endif
+
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/options.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/options.c
new file mode 100644
index 00000000..4fe7fe5a
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/options.c
@@ -0,0 +1,704 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/atomic.h"
+#include "mimalloc/prim.h"  // mi_prim_out_stderr
+
+#include <stdio.h>      // stdin/stdout
+#include <stdlib.h>     // abort
+
+static long mi_max_error_count   = 16; // stop outputting errors after this (use < 0 for no limit)
+static long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit)
+
+static void mi_add_stderr_output(void);
+
+int mi_version(void) mi_attr_noexcept {
+  return MI_MALLOC_VERSION;
+}
+
+
+// --------------------------------------------------------
+// Options
+// These can be accessed by multiple threads and may be
+// concurrently initialized, but an initializing data race
+// is ok since they resolve to the same value.
+// --------------------------------------------------------
+
+
+#define MI_OPTION(opt)                  mi_option_##opt, #opt, NULL
+#define MI_OPTION_LEGACY(opt,legacy)    mi_option_##opt, #opt, #legacy
+
+// Some options can be set at build time for statically linked libraries
+// (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`)
+//
+// This is useful if we cannot pass them as environment variables
+// (and setting them programmatically would be too late)
+
+#ifndef MI_DEFAULT_VERBOSE
+#define MI_DEFAULT_VERBOSE 0
+#endif
+
+#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT
+#define MI_DEFAULT_ARENA_EAGER_COMMIT 2
+#endif
+
+// in KiB
+#ifndef MI_DEFAULT_ARENA_RESERVE
+ #if (MI_INTPTR_SIZE>4)
+  #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L
+ #else
+  #define MI_DEFAULT_ARENA_RESERVE 128L*1024L
+ #endif
+#endif
+
+#ifndef MI_DEFAULT_ARENA_MAX_OBJECT_SIZE
+#define MI_DEFAULT_ARENA_MAX_OBJECT_SIZE   ((MI_SIZE_BITS * MI_ARENA_MAX_CHUNK_OBJ_SIZE)/MI_KiB)  /* 2 GiB (or 256 MiB on 32-bit), larger than this is alloc'd by the OS */
+#endif
+
+#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC
+#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0
+#endif
+
+#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES
+#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0
+#endif
+
+#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES
+#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0
+#endif
+
+#ifndef MI_DEFAULT_RESERVE_OS_MEMORY
+#define MI_DEFAULT_RESERVE_OS_MEMORY 0
+#endif
+
+#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE
+#if MI_GUARDED && !MI_DEBUG
+#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000
+#else
+#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0
+#endif
+#endif
+
+#ifndef MI_DEFAULT_PAGEMAP_COMMIT
+#if defined(__APPLE__)  // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access
+#define MI_DEFAULT_PAGEMAP_COMMIT 1
+#else
+#define MI_DEFAULT_PAGEMAP_COMMIT 0
+#endif
+#endif
+
+#ifndef MI_DEFAULT_PAGE_MAX_RECLAIM
+#define MI_DEFAULT_PAGE_MAX_RECLAIM  (-1)               // unlimited
+#endif
+
+#ifndef MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM
+#define MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM  32
+#endif
+
+#ifndef MI_DEFAULT_ALLOW_THP
+#if defined(__ANDROID__)
+#define MI_DEFAULT_ALLOW_THP  0
+#else
+#define MI_DEFAULT_ALLOW_THP  1
+#endif
+#endif
+
+// Static options
+static mi_option_desc_t mi_options[_mi_option_last] =
+{
+  // stable options
+#if MI_DEBUG || defined(MI_SHOW_ERRORS)
+  { 1, MI_OPTION_UNINIT, MI_OPTION(show_errors) },
+#else
+  { 0, MI_OPTION_UNINIT, MI_OPTION(show_errors) },
+#endif
+  { 0, MI_OPTION_UNINIT, MI_OPTION(show_stats) },
+  { MI_DEFAULT_VERBOSE, MI_OPTION_UNINIT, MI_OPTION(verbose) },
+
+  // some of the following options are experimental and not all combinations are allowed.
+  { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) },  
+  { MI_DEFAULT_ARENA_EAGER_COMMIT,
+       MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux)
+  { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) },        // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit)
+  { MI_DEFAULT_ALLOW_LARGE_OS_PAGES,
+       MI_OPTION_UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) },    // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
+  { MI_DEFAULT_RESERVE_HUGE_OS_PAGES,
+       MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages) },      // per 1GiB huge pages
+  {-1, MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages_at) },   // reserve huge pages at node N
+  { MI_DEFAULT_RESERVE_OS_MEMORY,
+       MI_OPTION_UNINIT, MI_OPTION(reserve_os_memory)     },      // reserve N KiB OS memory in advance (use `option_get_size`)
+  { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_cache) },   // cache N segments per thread
+  { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_page_reset) },      // reset page memory on free
+  { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) }, 
+  { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_reset) },   // reset segment memory on free (needs eager commit)
+  { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) },  
+  { 1000,MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) },  // purge delay in milli-seconds
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(use_numa_nodes) },           // 0 = use available numa nodes, otherwise use at most N nodes.
+  { 0,   MI_OPTION_UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) },           // 1 = do not use OS memory for allocation (but only reserved arenas)
+  { 100, MI_OPTION_UNINIT, MI_OPTION(os_tag) },                   // only apple specific for now but might serve more or less related purpose
+  { 32,  MI_OPTION_UNINIT, MI_OPTION(max_errors) },               // maximum errors that are output
+  { 32,  MI_OPTION_UNINIT, MI_OPTION(max_warnings) },             // maximum warnings that are output
+  { 10,  MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)},       // max. percentage of the abandoned segments to be reclaimed per try.
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)},           // release all OS memory on process exit; careful with dangling pointer or after-exit frees!
+  { MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)
+  { 1,   MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) },         // purge delay multiplier for arena's
+  { 1,   MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) },
+  { MI_DEFAULT_DISALLOW_ARENA_ALLOC,   MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)
+  { 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) },             // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.
+#if defined(MI_VISIT_ABANDONED)
+  { 1,   MI_OPTION_INITIALIZED, MI_OPTION(visit_abandoned) },     // allow visiting theap blocks in abandoned segments; requires taking locks during reclaim.
+#else
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(visit_abandoned) },
+#endif
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(guarded_min) },              // only used when building with MI_GUARDED: minimal rounded object size for guarded objects
+  { MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) },           // only used when building with MI_GUARDED: maximal rounded object size for guarded objects
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(guarded_precise) },          // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
+  { MI_DEFAULT_GUARDED_SAMPLE_RATE,
+         MI_OPTION_UNINIT, MI_OPTION(guarded_sample_rate)},       // 1 out of N allocations in the min/max range will be guarded (=4000)
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(guarded_sample_seed)},
+  { 10000, MI_OPTION_UNINIT, MI_OPTION(generic_collect) },        // collect theaps every N (=10000) generic allocation calls
+  { 0,   MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 = only reclaim into the originating theap, 1 = reclaim on free across theaps
+  { 2,   MI_OPTION_UNINIT, MI_OPTION(page_full_retain) },         // number of (small) pages to retain in the free page queues
+  { 4,   MI_OPTION_UNINIT, MI_OPTION(page_max_candidates) },      // max search to find a best page candidate
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(max_vabits) },               // max virtual address space bits
+  { MI_DEFAULT_PAGEMAP_COMMIT,
+         MI_OPTION_UNINIT, MI_OPTION(pagemap_commit) },           // commit the full pagemap upfront?
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(page_commit_on_demand) },    // commit pages on-demand (2 disables this only on overcommit systems (like Linux))
+  { MI_DEFAULT_PAGE_MAX_RECLAIM,
+         MI_OPTION_UNINIT, MI_OPTION(page_max_reclaim) },         // don't reclaim (small) pages of the same originating theap if we already own N pages in that size class
+  { MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM,
+         MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class
+  { MI_DEFAULT_ALLOW_THP,
+         MI_OPTION_UNINIT, MI_OPTION(allow_thp) },                // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) },       // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`).
+  { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE,   
+         MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) },    // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). 
+  { 0,   MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) },      // associate local numa node with an initial arena allocation
+};
+
+static void mi_option_init(mi_option_desc_t* desc);
+
+static bool mi_option_has_size_in_kib(mi_option_t option) {
+  return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve || 
+          option == mi_option_minimal_purge_size || option == mi_option_arena_max_object_size);
+}
+
+void _mi_options_init(void) {
+  // called on process load
+  for(int i = 0; i < _mi_option_last; i++ ) {
+    mi_option_t option = (mi_option_t)i;
+    long l = mi_option_get(option); MI_UNUSED(l); // initialize
+  }
+  mi_max_error_count = mi_option_get(mi_option_max_errors);
+  mi_max_warning_count = mi_option_get(mi_option_max_warnings);
+  #if MI_GUARDED
+  if (mi_option_get(mi_option_guarded_sample_rate) > 0) {
+    if (mi_option_is_enabled(mi_option_allow_large_os_pages)) {
+      mi_option_disable(mi_option_allow_large_os_pages);
+      _mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n");
+    }
+  }
+  #endif  
+}
+
+// called at actual process load, it should be safe to print now
+void _mi_options_post_init(void) {
+  mi_add_stderr_output(); // now it safe to use stderr for output
+  if (mi_option_is_enabled(mi_option_verbose)) { mi_options_print(); }
+}
+
+#define mi_stringifyx(str)  #str                // and stringify
+#define mi_stringify(str)   mi_stringifyx(str)  // expand
+
+mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept
+{
+  // show version
+  const int vermajor = MI_MALLOC_VERSION/10000;
+  const int verminor = (MI_MALLOC_VERSION%10000)/100;
+  const int verpatch = (MI_MALLOC_VERSION%100);
+  _mi_fprintf(out, arg, "v%i.%i.%i%s%s (built on %s, %s)\n", vermajor, verminor, verpatch,
+      #if defined(MI_CMAKE_BUILD_TYPE)
+      ", " mi_stringify(MI_CMAKE_BUILD_TYPE)
+      #else
+      ""
+      #endif
+      ,
+      #if defined(MI_GIT_DESCRIBE)
+      ", git " mi_stringify(MI_GIT_DESCRIBE)
+      #else
+      ""
+      #endif
+      , __DATE__, __TIME__);
+
+  // show options
+  for (int i = 0; i < _mi_option_last; i++) {
+    mi_option_t option = (mi_option_t)i;
+    long l = mi_option_get(option); MI_UNUSED(l); // possibly initialize
+    mi_option_desc_t* desc = &mi_options[option];
+    _mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : ""));
+  }
+
+  // show build configuration
+  _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG );
+  _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE );
+  _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL);
+  #if MI_GUARDED
+  _mi_fprintf(out, arg, "guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled");
+  #endif
+  #if MI_TSAN
+  _mi_fprintf(out, arg, "thread santizer enabled\n");
+  #endif
+}
+
+mi_decl_export void mi_options_print(void) mi_attr_noexcept {
+  mi_options_print_out(NULL, NULL);
+}
+
+long _mi_option_get_fast(mi_option_t option) {
+  mi_assert(option >= 0 && option < _mi_option_last);
+  mi_option_desc_t* desc = &mi_options[option];
+  mi_assert(desc->option == option);  // index should match the option
+  //mi_assert(desc->init != MI_OPTION_UNINIT);
+  return desc->value;
+}
+
+
+mi_decl_nodiscard long mi_option_get(mi_option_t option) {
+  mi_assert(option >= 0 && option < _mi_option_last);
+  if (option < 0 || option >= _mi_option_last) return 0;
+  mi_option_desc_t* desc = &mi_options[option];
+  mi_assert(desc->option == option);  // index should match the option
+  if mi_unlikely(desc->init == MI_OPTION_UNINIT) {
+    mi_option_init(desc);
+  }
+  return desc->value;
+}
+
+mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long max) {
+  long x = mi_option_get(option);
+  return (x < min ? min : (x > max ? max : x));
+}
+
+mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {
+  const long x = mi_option_get(option);
+  size_t size = (x < 0 ? 0 : (size_t)x);
+  if (mi_option_has_size_in_kib(option)) {
+    if (mi_mul_overflow(size, MI_KiB, &size)) {
+      size = MI_MAX_ALLOC_SIZE;
+    }
+  }
+  return size;
+}
+
+void mi_option_set(mi_option_t option, long value) {
+  mi_assert(option >= 0 && option < _mi_option_last);
+  if (option < 0 || option >= _mi_option_last) return;
+  mi_option_desc_t* desc = &mi_options[option];
+  mi_assert(desc->option == option);  // index should match the option
+  desc->value = value;
+  desc->init = MI_OPTION_INITIALIZED;
+  // ensure min/max range; be careful to not recurse.
+  if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) {
+    mi_option_set(mi_option_guarded_max, value);
+  }
+  else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) {
+    mi_option_set(mi_option_guarded_min, value);
+  }
+}
+
+void mi_option_set_default(mi_option_t option, long value) {
+  mi_assert(option >= 0 && option < _mi_option_last);
+  if (option < 0 || option >= _mi_option_last) return;
+  mi_option_desc_t* desc = &mi_options[option];
+  if (desc->init != MI_OPTION_INITIALIZED) {
+    desc->value = value;
+  }
+}
+
+mi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option) {
+  return (mi_option_get(option) != 0);
+}
+
+void mi_option_set_enabled(mi_option_t option, bool enable) {
+  mi_option_set(option, (enable ? 1 : 0));
+}
+
+void mi_option_set_enabled_default(mi_option_t option, bool enable) {
+  mi_option_set_default(option, (enable ? 1 : 0));
+}
+
+void mi_option_enable(mi_option_t option) {
+  mi_option_set_enabled(option,true);
+}
+
+void mi_option_disable(mi_option_t option) {
+  mi_option_set_enabled(option,false);
+}
+
+static void mi_cdecl mi_out_stderr(const char* msg, void* arg) {
+  MI_UNUSED(arg);
+  if (msg != NULL && msg[0] != 0) {
+    _mi_prim_out_stderr(msg);
+  }
+}
+
+// Since an output function can be registered earliest in the `main`
+// function we also buffer output that happens earlier. When
+// an output function is registered it is called immediately with
+// the output up to that point.
+#ifndef MI_MAX_DELAY_OUTPUT
+#define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024))
+#endif
+static char out_buf[MI_MAX_DELAY_OUTPUT+1];
+static _Atomic(size_t) out_len;
+static mi_lock_t out_buf_lock = MI_LOCK_INITIALIZER;
+
+static void mi_cdecl mi_out_buf(const char* msg, void* arg) {
+  MI_UNUSED(arg);
+  if (msg==NULL) return;
+  if (mi_atomic_load_acquire(&out_len)>=MI_MAX_DELAY_OUTPUT) return;
+  size_t n = _mi_strlen(msg);
+  if (n==0 || n >= MI_MAX_DELAY_OUTPUT) return;
+  // copy msg into the buffer
+  mi_lock(&out_buf_lock) {
+    const size_t start = mi_atomic_add_acq_rel(&out_len, n);
+    if (start < MI_MAX_DELAY_OUTPUT) {
+      // check bound
+      if (start+n >= MI_MAX_DELAY_OUTPUT) {
+        n = MI_MAX_DELAY_OUTPUT-start-1;
+      }
+      _mi_memcpy(&out_buf[start], msg, n);
+    }
+  }
+}
+
+static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) {
+  if (out==NULL) return;
+  // claim (if `no_more_buf == true`, no more output will be added after this point)
+  mi_lock(&out_buf_lock) {
+    size_t count = mi_atomic_add_acq_rel(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1));
+    // and output the current contents
+    if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT;
+    out_buf[count] = 0;
+    out(out_buf,arg);
+    if (!no_more_buf) {
+      out_buf[count] = '\n'; // if continue with the buffer, insert a newline
+    }
+  }
+}
+
+
+// Once this module is loaded, switch to this routine
+// which outputs to stderr and the delayed output buffer.
+static void mi_cdecl mi_out_buf_stderr(const char* msg, void* arg) {
+  mi_out_stderr(msg,arg);
+  mi_out_buf(msg,arg);
+}
+
+
+
+// --------------------------------------------------------
+// Default output handler
+// --------------------------------------------------------
+
+// The program should only install a single output handler from a single thread
+// since otherwise the argument and output function may not match.
+static _Atomic(void*) mi_out_default; // = // is `mi_output_fun*` (but some platforms don't support atomic function pointers)
+static _Atomic(void*) mi_out_arg; // = NULL
+
+static mi_output_fun* mi_out_get_default(void** parg) {
+  mi_output_fun* const out = (mi_output_fun*)mi_atomic_load_ptr_acquire(void,&mi_out_default);
+  if (parg != NULL) { *parg = mi_atomic_load_ptr_acquire(void,&mi_out_arg); }
+  return (out == NULL ? &mi_out_buf : out);
+}
+
+void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept {
+  mi_atomic_store_ptr_release(void,&mi_out_default, (void*)(out == NULL ? &mi_out_stderr : out)); // stop using the delayed output buffer
+  mi_atomic_store_ptr_release(void,&mi_out_arg, arg);
+  if (out!=NULL) { mi_out_buf_flush(out,true,arg); }        // output all the delayed output now
+}
+
+// add stderr to the delayed output after the module is loaded
+static void mi_add_stderr_output(void) {
+  mi_assert_internal(mi_out_default == NULL);
+  mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr
+  mi_atomic_store_ptr_release(void,&mi_out_default,(void*)&mi_out_buf_stderr);  // and add stderr to the delayed output
+  mi_atomic_store_ptr_release(void,&mi_out_arg,NULL);
+}
+
+// --------------------------------------------------------
+// Messages, all end up calling `_mi_fputs`.
+// --------------------------------------------------------
+static _Atomic(size_t) error_count;   // = 0;  // when >= max_error_count stop emitting errors
+static _Atomic(size_t) warning_count; // = 0;  // when >= max_warning_count stop emitting warnings
+
+// When overriding malloc, we may recurse into mi_vfprintf if an allocation
+// inside the C runtime causes another message.
+// In some cases (like on macOS) the loader already allocates which
+// calls into mimalloc; if we then access thread locals (like `recurse`)
+// this may crash as the access may call _tlv_bootstrap that tries to
+// (recursively) invoke malloc again to allocate space for the thread local
+// variables on demand. This is why we use a _mi_preloading test on such
+// platforms. However, C code generator may move the initial thread local address
+// load before the `if` and we therefore split it out in a separate function.
+static mi_decl_thread bool recurse = false;
+
+static mi_decl_noinline bool mi_recurse_enter_prim(void) {
+  if (recurse) return false;
+  recurse = true;
+  return true;
+}
+
+static mi_decl_noinline void mi_recurse_exit_prim(void) {
+  recurse = false;
+}
+
+static bool mi_recurse_enter(void) {
+  #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)
+  if (_mi_preloading()) return false;
+  #endif
+  return mi_recurse_enter_prim();
+}
+
+static void mi_recurse_exit(void) {
+  #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)
+  if (_mi_preloading()) return;
+  #endif
+  mi_recurse_exit_prim();
+}
+
+void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {
+  if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // TODO: use mi_out_stderr for stderr?
+    if (!mi_recurse_enter()) return;
+    out = mi_out_get_default(&arg);
+    if (prefix != NULL) out(prefix, arg);
+    out(message, arg);
+    mi_recurse_exit();
+  }
+  else {
+    if (prefix != NULL) out(prefix, arg);
+    out(message, arg);
+  }
+}
+
+// Define our own limited `fprintf` that avoids memory allocation.
+// We do this using `_mi_vsnprintf` with a limited buffer.
+static void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) {
+  char buf[992];
+  if (fmt==NULL) return;
+  if (!mi_recurse_enter()) return;
+  _mi_vsnprintf(buf, sizeof(buf)-1, fmt, args);
+  mi_recurse_exit();
+  _mi_fputs(out,arg,prefix,buf);
+}
+
+void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {
+  va_list args;
+  va_start(args,fmt);
+  mi_vfprintf(out,arg,NULL,fmt,args);
+  va_end(args);
+}
+
+static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {
+  if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) {
+    char tprefix[64];
+    _mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id());
+    mi_vfprintf(out, arg, tprefix, fmt, args);
+  }
+  else {
+    mi_vfprintf(out, arg, prefix, fmt, args);
+  }
+}
+
+void _mi_raw_message(const char* fmt, ...) {
+  va_list args;
+  va_start(args, fmt);
+  mi_vfprintf(NULL, NULL, NULL, fmt, args);
+  va_end(args);
+}
+
+void _mi_message(const char* fmt, ...) {
+  va_list args;
+  va_start(args, fmt);
+  mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args);
+  va_end(args);
+}
+
+void _mi_trace_message(const char* fmt, ...) {
+  if (mi_option_get(mi_option_verbose) <= 1) return;  // only with verbose level 2 or higher
+  va_list args;
+  va_start(args, fmt);
+  mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args);
+  va_end(args);
+}
+
+void _mi_verbose_message(const char* fmt, ...) {
+  if (!mi_option_is_enabled(mi_option_verbose)) return;
+  va_list args;
+  va_start(args,fmt);
+  mi_vfprintf(NULL, NULL, "mimalloc: ", fmt, args);
+  va_end(args);
+}
+
+static void mi_show_error_message(const char* fmt, va_list args) {
+  if (!mi_option_is_enabled(mi_option_verbose)) {
+    if (!mi_option_is_enabled(mi_option_show_errors)) return;
+    if (mi_max_error_count >= 0 && (long)mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return;
+  }
+  mi_vfprintf_thread(NULL, NULL, "mimalloc: error: ", fmt, args);
+}
+
+void _mi_warning_message(const char* fmt, ...) {
+  if (!mi_option_is_enabled(mi_option_verbose)) {
+    if (!mi_option_is_enabled(mi_option_show_errors)) return;
+    if (mi_max_warning_count >= 0 && (long)mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return;
+  }
+  va_list args;
+  va_start(args,fmt);
+  mi_vfprintf_thread(NULL, NULL, "mimalloc: warning: ", fmt, args);
+  va_end(args);
+}
+
+
+#if MI_DEBUG
+mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept {
+  _mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n  assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion);
+  abort();
+}
+#endif
+
+// --------------------------------------------------------
+// Errors
+// --------------------------------------------------------
+
+static mi_error_fun* volatile  mi_error_handler; // = NULL
+static _Atomic(void*) mi_error_arg;     // = NULL
+
+static void mi_error_default(int err) {
+  MI_UNUSED(err);
+#if (MI_DEBUG>0)
+  if (err==EFAULT) {
+    #ifdef _MSC_VER
+    __debugbreak();
+    #endif
+    abort();
+  }
+#endif
+#if (MI_SECURE>0)
+  if (err==EFAULT) {  // abort on serious errors in secure mode (corrupted meta-data)
+    abort();
+  }
+#endif
+#if defined(MI_XMALLOC)
+  if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode
+    abort();
+  }
+#endif
+}
+
+void mi_register_error(mi_error_fun* fun, void* arg) {
+  mi_error_handler = fun;  // can be NULL
+  mi_atomic_store_ptr_release(void,&mi_error_arg, arg);
+}
+
+void _mi_error_message(int err, const char* fmt, ...) {
+  // show detailed error message
+  va_list args;
+  va_start(args, fmt);
+  mi_show_error_message(fmt, args);
+  va_end(args);
+  // and call the error handler which may abort (or return normally)
+  if (mi_error_handler != NULL) {
+    mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg));
+  }
+  else {
+    mi_error_default(err);
+  }
+}
+
+// --------------------------------------------------------
+// Initialize options by checking the environment
+// --------------------------------------------------------
+
+// TODO: implement ourselves to reduce dependencies on the C runtime
+#include <stdlib.h> // strtol
+#include <string.h> // strstr
+
+
+static void mi_option_init(mi_option_desc_t* desc) {
+  // Read option value from the environment
+  char s[64 + 1];
+  char buf[64+1];
+  _mi_strlcpy(buf, "mimalloc_", sizeof(buf));
+  _mi_strlcat(buf, desc->name, sizeof(buf));
+  bool found = _mi_getenv(buf, s, sizeof(s));
+  if (!found && desc->legacy_name != NULL) {
+    _mi_strlcpy(buf, "mimalloc_", sizeof(buf));
+    _mi_strlcat(buf, desc->legacy_name, sizeof(buf));
+    found = _mi_getenv(buf, s, sizeof(s));
+    if (found) {
+      _mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name);
+    }
+  }
+
+  if (found) {
+    size_t len = _mi_strnlen(s, sizeof(buf) - 1);
+    for (size_t i = 0; i < len; i++) {
+      buf[i] = _mi_toupper(s[i]);
+    }
+    buf[len] = 0;
+    if (buf[0] == 0 || _mi_streq(buf,"1") || _mi_streq(buf,"TRUE") || _mi_streq(buf,"YES") || _mi_streq(buf,"ON")) {
+      desc->value = 1;
+      desc->init = MI_OPTION_INITIALIZED;
+    }
+    else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) {      
+      desc->value = 0;
+      desc->init = MI_OPTION_INITIALIZED;
+    }
+    else {
+      char* end = buf;
+      long value = strtol(buf, &end, 10);
+      if (mi_option_has_size_in_kib(desc->option)) {
+        // this option is interpreted in KiB to prevent overflow of `long` for large allocations
+        // (long is 32-bit on 64-bit windows, which allows for 4TiB max.)
+        size_t size = (value < 0 ? 0 : (size_t)value);
+        bool overflow = false;
+        if (*end == 'K') { end++; }
+        else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; }
+        else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; }
+        else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; }
+        else { size = (size + MI_KiB - 1) / MI_KiB; }
+        if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB
+        else if (*end == 'B') { end++; }                  // Kb, Mb, Gb, Tb
+        if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); }
+        value = (size > LONG_MAX ? LONG_MAX : (long)size);
+      }
+      if (*end == 0) {
+        mi_option_set(desc->option, value);
+      }
+      else {
+        // set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose.
+        desc->init = MI_OPTION_DEFAULTED;
+        if (desc->option == mi_option_verbose && desc->value == 0) {
+          // if the 'mimalloc_verbose' env var has a bogus value we'd never know
+          // (since the value defaults to 'off') so in that case briefly enable verbose
+          desc->value = 1;
+          _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name);
+          desc->value = 0;
+        }
+        else {
+          _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name);
+        }
+      }
+    }
+    mi_assert_internal(desc->init != MI_OPTION_UNINIT);
+  }
+  else if (!_mi_preloading()) {
+    desc->init = MI_OPTION_DEFAULTED;
+  }
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/os.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/os.c
new file mode 100644
index 00000000..febecb0c
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/os.c
@@ -0,0 +1,923 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/atomic.h"
+#include "mimalloc/prim.h"
+
+/* -----------------------------------------------------------
+  Initialization.
+----------------------------------------------------------- */
+#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB
+#if MI_INTPTR_SIZE < 8
+#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB   4*MI_MiB    // 4 GiB
+#else
+#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB   32*MI_MiB   // 32 GiB
+#endif
+#endif
+
+static mi_os_mem_config_t mi_os_mem_config = {
+  4096,     // page size
+  0,        // large page size (usually 2MiB)
+  4096,     // allocation granularity
+  MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB,
+  MI_MAX_VABITS, // in `bits.h`
+  true,     // has overcommit?  (if true we use MAP_NORESERVE on mmap systems)
+  false,    // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
+  true,     // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)
+  false     // has transparent huge pages? (if true we purge in (aligned) large page size chunks only to not fragment such pages)
+};
+
+bool _mi_os_has_overcommit(void) {
+  return mi_os_mem_config.has_overcommit;
+}
+
+bool _mi_os_has_virtual_reserve(void) {
+  return mi_os_mem_config.has_virtual_reserve;
+}
+
+
+// OS (small) page size
+size_t _mi_os_page_size(void) {
+  return mi_os_mem_config.page_size;
+}
+
+// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB)
+size_t _mi_os_large_page_size(void) {
+  return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size());
+}
+
+// minimal purge size. Can be larger than the page size if transparent huge pages are enabled.
+size_t _mi_os_minimal_purge_size(void) {
+  size_t minsize = mi_option_get_size(mi_option_minimal_purge_size);
+  if (minsize != 0) {
+    return _mi_align_up(minsize, _mi_os_page_size());
+  }
+  else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) {
+    return _mi_os_large_page_size();
+  }
+  else {
+    return _mi_os_page_size();
+  }
+}
+
+size_t _mi_os_guard_page_size(void) {
+  const size_t gsize = _mi_os_page_size();
+  mi_assert(gsize <= (MI_ARENA_SLICE_SIZE/4)); // issue #1166
+  return gsize;
+}
+
+size_t _mi_os_virtual_address_bits(void) {
+  const size_t vbits = mi_os_mem_config.virtual_address_bits;
+  mi_assert(vbits <= MI_MAX_VABITS);
+  return vbits;
+}
+
+bool _mi_os_canuse_large_page(size_t size, size_t alignment) {
+  // if we have access, check the size and alignment requirements
+  if (mi_os_mem_config.large_page_size == 0) return false;
+  return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0);
+}
+
+// round to a good OS allocation size (bounded by max 12.5% waste)
+size_t _mi_os_good_alloc_size(size_t size) {
+  size_t align_size;
+  if (size < 512*MI_KiB) align_size = _mi_os_page_size();
+  else if (size < 2*MI_MiB) align_size = 64*MI_KiB;
+  else if (size < 8*MI_MiB) align_size = 256*MI_KiB;
+  else if (size < 32*MI_MiB) align_size = 1*MI_MiB;
+  else align_size = 4*MI_MiB;
+  if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow?
+  return _mi_align_up(size, align_size);
+}
+
+void _mi_os_init(void) {
+  _mi_prim_mem_init(&mi_os_mem_config);
+}
+
+
+/* -----------------------------------------------------------
+  Util
+-------------------------------------------------------------- */
+bool _mi_os_decommit(void* addr, size_t size);
+bool _mi_os_commit(void* addr, size_t size, bool* is_zero);
+
+// On systems with enough virtual address bits, we can do efficient aligned allocation by using
+// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address
+// space (64TiB) we use this technique. (but see issue #939)
+#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID)
+
+// Return a MI_HINT_ALIGN (4MiB) aligned address that is probably available.
+// If this returns NULL, the OS will determine the address but on some OS's that may not be
+// properly aligned which can be more costly as it needs to be adjusted afterwards.
+// For a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization;
+// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses
+//  in the middle of the 2TiB - 6TiB address range (see issue #372))
+
+#define MI_HINT_ALIGN ((uintptr_t)4 << 20)  // 4MiB alignment
+#define MI_HINT_BASE  ((uintptr_t)2 << 40)  // 2TiB start
+#define MI_HINT_AREA  ((uintptr_t)4 << 40)  // upto (2+4) 6TiB  (since before win8 there is "only" 8TiB available to processes)
+#define MI_HINT_MAX   ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)
+
+void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
+{
+  static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0
+
+  // todo: perhaps only do alignment hints if THP is enabled?
+  if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL;
+  if (mi_os_mem_config.virtual_address_bits < 46) return NULL;  // < 64TiB virtual address space
+  size = _mi_align_up(size, MI_HINT_ALIGN);
+  if (size > 16*MI_GiB) return NULL;  // guarantee the chance of fixed valid address is at least 1/(MI_HINT_AREA / 1<<34)
+  size += MI_HINT_ALIGN;              // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.
+
+  uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);
+  if (hint == 0 || hint > MI_HINT_MAX) {   // wrap or initialize
+    uintptr_t init = MI_HINT_BASE;
+    #if (MI_SECURE>=1 || defined(NDEBUG))  // security: randomize start of aligned allocations unless in debug mode
+    mi_theap_t* const theap = _mi_theap_default();     // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
+    if (!mi_theap_is_initialized(theap)) return NULL;  // no hint as we lack randomness at this point
+    const uintptr_t r = _mi_theap_random_next(theap);
+    init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA);  // (randomly 20 bits)*4MiB == 0 to 4TiB
+    #endif
+    uintptr_t expected = hint + size;
+    mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
+    hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all
+  }
+  mi_assert_internal(hint%MI_HINT_ALIGN == 0);
+  if (hint%try_alignment != 0) return NULL;
+  return (void*)hint;
+}
+#else
+void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
+   MI_UNUSED(try_alignment); MI_UNUSED(size);
+   return NULL;
+}
+#endif
+
+
+/* -----------------------------------------------------------
+  Guard page allocation
+----------------------------------------------------------- */
+
+// In secure mode, return the size of a guard page, otherwise 0
+size_t _mi_os_secure_guard_page_size(void) {
+  #if MI_SECURE > 0
+  return _mi_os_guard_page_size();
+  #else
+  return 0;
+  #endif
+}
+
+// In secure mode, try to decommit an area and output a warning if this fails.
+bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {
+  if (addr == NULL) return true;
+  #if MI_SECURE > 0
+  bool ok = false;
+  if (!memid.is_pinned) {
+    mi_arena_t* const arena = mi_memid_arena(memid);
+    if (arena != NULL && arena->commit_fun != NULL) {
+      ok = (*(arena->commit_fun))(false /* decommit */, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);
+    }
+    else {
+      ok = _mi_os_decommit(addr, _mi_os_secure_guard_page_size());
+    }
+  }
+  if (!ok) {
+    _mi_error_message(EINVAL, "secure level %d, but failed to commit guard page (at %p of size %zu)\n", MI_SECURE, addr, _mi_os_secure_guard_page_size());
+  }
+  return ok;
+  #else
+  MI_UNUSED(memid);
+  return true;
+  #endif
+}
+
+// In secure mode, try to decommit an area and output a warning if this fails.
+bool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid) {
+  return _mi_os_secure_guard_page_set_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
+}
+
+// In secure mode, try to recommit an area
+bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) {
+  if (addr == NULL) return true;
+  #if MI_SECURE > 0
+  if (!memid.is_pinned) {
+    mi_arena_t* const arena = mi_memid_arena(memid);
+    if (arena != NULL && arena->commit_fun != NULL) {
+      return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);
+    }
+    else {
+      return _mi_os_commit(addr, _mi_os_secure_guard_page_size(), NULL);
+    }
+  }
+  #else
+  MI_UNUSED(memid);
+  #endif
+  return true;
+}
+
+// In secure mode, try to recommit an area
+bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) {
+  return _mi_os_secure_guard_page_reset_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
+}
+
+
+/* -----------------------------------------------------------
+  Free memory
+-------------------------------------------------------------- */
+
+static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc);
+
+static void mi_os_prim_free(void* addr, size_t size, size_t commit_size, mi_subproc_t* subproc) {
+  mi_assert_internal((size % _mi_os_page_size()) == 0);
+  if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr)
+  int err = _mi_prim_free(addr, size);  // allow size==0 (issue #1041)
+  if (err != 0) {
+    _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
+  }
+  if (subproc == NULL) { subproc = _mi_subproc(); } // from `mi_arenas_unsafe_destroy` we pass subproc_main explicitly as we can no longer use the theap pointer
+  if (commit_size > 0) {
+    mi_subproc_stat_decrease(subproc, committed, commit_size);
+  }
+  mi_subproc_stat_decrease(subproc, reserved, size);
+}
+
+void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc /* can be NULL */) {
+  if (mi_memkind_is_os(memid.memkind)) {
+    size_t csize = memid.mem.os.size;
+    if (csize==0) { csize = _mi_os_good_alloc_size(size); }
+    mi_assert_internal(csize >= size);
+    size_t commit_size = (still_committed ? csize : 0);
+    void* base = addr;
+    // different base? (due to alignment)
+    if (memid.mem.os.base != base) {
+      mi_assert(memid.mem.os.base <= addr);
+      base = memid.mem.os.base;
+      const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base;
+      if (memid.mem.os.size==0) {
+        csize += diff;
+      }
+      if (still_committed) {
+        commit_size -= diff;  // the (addr-base) part was already un-committed
+      }
+    }
+    // free it
+    if (memid.memkind == MI_MEM_OS_HUGE) {
+      mi_assert(memid.is_pinned);
+      mi_os_free_huge_os_pages(base, csize, subproc);
+    }
+    else {
+      mi_os_prim_free(base, csize, (still_committed ? commit_size : 0), subproc);
+    }
+  }
+  else {
+    // nothing to do
+    mi_assert(memid.memkind < MI_MEM_OS);
+  }
+}
+
+void  _mi_os_free(void* p, size_t size, mi_memid_t memid) {
+  _mi_os_free_ex(p, size, true, memid, NULL);
+}
+
+
+/* -----------------------------------------------------------
+   Primitive allocation from the OS.
+-------------------------------------------------------------- */
+
+// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
+// Also `hint_addr` is a hint and may be ignored.
+static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
+  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
+  mi_assert_internal(is_zero != NULL);
+  mi_assert_internal(is_large != NULL);
+  if (size == 0) return NULL;
+  if (!commit) { allow_large = false; }
+  if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning
+
+  // try to align along large OS page size for larger allocations
+  const size_t large_page_size = mi_os_mem_config.large_page_size;
+  if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && _mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) {
+    try_alignment = large_page_size;
+  }
+
+  *is_zero = false;
+  void* p = NULL;
+  int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p);
+  if (err != 0) {
+    _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);
+  }
+
+  mi_os_stat_counter_increase(mmap_calls, 1);
+  if (p != NULL) {
+    mi_os_stat_increase(reserved, size);
+    if (commit) {
+      mi_os_stat_increase(committed, size);
+      // seems needed for asan (or `mimalloc-test-api` fails)
+      #ifdef MI_TRACK_ASAN
+      if (*is_zero) { mi_track_mem_defined(p,size); }
+               else { mi_track_mem_undefined(p,size); }
+      #endif
+    }
+  }
+  return p;
+}
+
+static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
+  return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero);
+}
+
+
+// Primitive aligned allocation from the OS.
+// This function guarantees the allocated memory is aligned.
+static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {
+  mi_assert_internal(memid!=NULL);
+  mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
+  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
+  *memid = _mi_memid_none();
+  if (!commit) allow_large = false;
+  if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
+  size = _mi_align_up(size, _mi_os_page_size());
+
+  // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size.
+  const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4);
+
+  bool os_is_large = false;
+  bool os_is_zero = false;
+  void*  os_base = NULL;
+  size_t os_size = size;
+  void* p = NULL;
+  if (try_direct_alloc) {
+    p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero);
+  }
+
+  // aligned already?
+  if (p != NULL && _mi_is_aligned(p,alignment)) {
+    os_base = p;
+  }
+  else {
+    // if not aligned, free it, overallocate, and unmap around it
+    #if !MI_TRACK_ASAN
+    if (try_direct_alloc) {
+      _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
+    }
+    #endif
+    if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0), NULL); }
+    if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
+    const size_t over_size = size + alignment;
+
+    if (!mi_os_mem_config.has_partial_free) {  // win32 virtualAlloc cannot free parts of an allocated block
+      // over-allocate uncommitted (virtual) memory
+      p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);
+      if (p == NULL) return NULL;
+
+      // set p to the aligned part in the full region
+      // note: Windows VirtualFree needs the actual base pointer
+      // this is handled though by having the `base` field in the memid
+      os_base = p; // remember the base
+      os_size = over_size;
+      p = _mi_align_up_ptr(p, alignment);
+
+      // explicitly commit only the aligned part
+      if (commit) {
+        if (!_mi_os_commit(p, size, NULL)) {
+          mi_os_prim_free(os_base, over_size, 0, NULL);
+          return NULL;
+        }
+      }
+    }
+    else  { // mmap can free inside an allocation
+      // overallocate...
+      p = mi_os_prim_alloc(over_size, 1, commit, false, &os_is_large, &os_is_zero);
+      if (p == NULL) return NULL;
+
+      // and selectively unmap parts around the over-allocated area.
+      void* const aligned_p = _mi_align_up_ptr(p, alignment);
+      const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
+      const size_t mid_size = _mi_align_up(size, _mi_os_page_size());
+      const size_t post_size = over_size - pre_size - mid_size;
+      mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
+      if (pre_size > 0)  { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0), NULL); }
+      if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), NULL); }
+      // we can return the aligned pointer on `mmap` systems
+      p = aligned_p;
+      os_base = aligned_p; // since we freed the pre part, `*base == p`.
+      os_size = mid_size;
+    }
+  }
+
+  mi_assert_internal(p != NULL && os_base != NULL && _mi_is_aligned(p,alignment));
+  mi_assert_internal(os_base <= p && size <= os_size);
+  *memid = _mi_memid_create_os(os_base,os_size,commit,os_is_zero,os_is_large);
+  return p;
+}
+
+
+/* -----------------------------------------------------------
+  OS API: alloc and alloc_aligned
+----------------------------------------------------------- */
+
+void* _mi_os_alloc(size_t size, mi_memid_t* memid) {
+  *memid = _mi_memid_none();
+  if (size == 0) return NULL;
+  size = _mi_os_good_alloc_size(size);
+  bool os_is_large = false;
+  bool os_is_zero  = false;
+  void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero);
+  if (p == NULL) return NULL;
+
+  *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large);
+  mi_assert_internal(memid->mem.os.size >= size);
+  mi_assert_internal(memid->initially_committed);
+  return p;
+}
+
+void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)
+{
+  MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings
+  *memid = _mi_memid_none();
+  if (size == 0) return NULL;
+  size = _mi_os_good_alloc_size(size);
+  alignment = _mi_align_up(alignment, _mi_os_page_size());
+
+  void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid );
+  if (p == NULL) return NULL;
+
+  mi_assert_internal(memid->mem.os.size >= size);
+  mi_assert_internal(_mi_is_aligned(p,alignment));
+  if (commit) { mi_assert_internal(memid->initially_committed); }
+  return p;
+}
+
+
+mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) {
+  if (p==NULL || size==0) return p;
+  // ensure committed
+  if (!memid->initially_committed) {
+    bool is_zero = false;
+    if (!_mi_os_commit(p, size, &is_zero)) {
+      _mi_os_free(p, size, *memid);
+      return NULL;
+    }
+    memid->initially_committed = true;
+  }
+  // ensure zero'd
+  if (memid->initially_zero) return p;
+  _mi_memzero_aligned(p,size);
+  memid->initially_zero = true;
+  return p;
+}
+
+void*  _mi_os_zalloc(size_t size, mi_memid_t* memid) {
+  void* p = _mi_os_alloc(size,memid);
+  return mi_os_ensure_zero(p, size, memid);
+}
+
+/* -----------------------------------------------------------
+  OS aligned allocation with an offset. This is used
+  for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc
+  page where the object can be aligned at an offset from the start of the segment.
+  As we may need to overallocate, we need to free such pointers using `mi_free_aligned`
+  to use the actual start of the memory region.
+----------------------------------------------------------- */
+
+void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {
+  mi_assert(offset <= size);
+  mi_assert((alignment % _mi_os_page_size()) == 0);
+  *memid = _mi_memid_none();
+  if (offset > size) return NULL;
+  if (offset == 0) {
+    // regular aligned allocation
+    return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);
+  }
+  else {
+    // overallocate to align at an offset
+    const size_t extra = _mi_align_up(offset, alignment) - offset;
+    if (size >= SIZE_MAX - extra) return NULL;  // too large
+    const size_t oversize = size + extra;
+    void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid);
+    if (start == NULL) return NULL;
+
+    void* const p = (uint8_t*)start + extra;
+    mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));
+    // decommit the overallocation at the start
+    if (commit && extra >= _mi_os_page_size()) {
+      _mi_os_decommit(start, extra);
+    }
+    return p;
+  }
+}
+
+/* -----------------------------------------------------------
+  OS memory API: reset, commit, decommit, protect, unprotect.
+----------------------------------------------------------- */
+
+// OS page align within a given area, either conservative (pages inside the area only),
+// or not (straddling pages outside the area is possible)
+static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
+  mi_assert(addr != NULL && size > 0);
+  if (newsize != NULL) *newsize = 0;
+  if (size == 0 || addr == NULL) return NULL;
+
+  // page align conservatively within the range, or liberally straddling pages outside the range
+  void* start = (conservative ? _mi_align_up_ptr(addr, _mi_os_page_size())
+                              : _mi_align_down_ptr(addr, _mi_os_page_size()));
+  void* end   = (conservative ? _mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
+                              : _mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
+  ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
+  if (diff <= 0) return NULL;
+
+  mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size));
+  if (newsize != NULL) *newsize = (size_t)diff;
+  return start;
+}
+
+static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
+  return mi_os_page_align_areax(true, addr, size, newsize);
+}
+
+bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {
+  if (is_zero != NULL) { *is_zero = false; }
+  mi_os_stat_counter_increase(commit_calls, 1);
+
+  // page align range
+  size_t csize;
+  void* start = mi_os_page_align_areax(false /* conservative? */, addr, size, &csize);
+  if (csize == 0) return true;
+
+  // commit
+  bool os_is_zero = false;
+  int err = _mi_prim_commit(start, csize, &os_is_zero);
+  if (err != 0) {
+    _mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
+    return false;
+  }
+  if (os_is_zero && is_zero != NULL) {
+    *is_zero = true;
+    mi_assert_expensive(mi_mem_is_zero(start, csize));
+  }
+  // note: the following seems required for asan (otherwise `mimalloc-test-stress` fails)
+  #ifdef MI_TRACK_ASAN
+  if (os_is_zero) { mi_track_mem_defined(start,csize); }
+             else { mi_track_mem_undefined(start,csize); }
+  #endif
+  mi_os_stat_increase(committed, stat_size);  // use size for precise commit vs. decommit
+  return true;
+}
+
+bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
+  return _mi_os_commit_ex(addr, size, is_zero, size);
+}
+
+static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
+  mi_assert_internal(needs_recommit!=NULL);
+
+  // page align
+  size_t csize;
+  void* start = mi_os_page_align_area_conservative(addr, size, &csize);
+  if (csize == 0) return true;
+
+  // decommit
+  *needs_recommit = true;
+  int err = _mi_prim_decommit(start,csize,needs_recommit);
+  if (err != 0) {
+    _mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
+  }
+  else if (*needs_recommit) {
+    mi_os_stat_decrease(committed, stat_size);
+  }
+  mi_assert_internal(err == 0);
+  return (err == 0);
+}
+
+bool _mi_os_decommit(void* addr, size_t size) {
+  bool needs_recommit;
+  return mi_os_decommit_ex(addr, size, &needs_recommit, size);
+}
+
+
+// Signal to the OS that the address range is no longer in use
+// but may be used later again. This will release physical memory
+// pages and reduce swapping while keeping the memory committed.
+// We page align to a conservative area inside the range to reset.
+bool _mi_os_reset(void* addr, size_t size) {
+  // page align conservatively within the range
+  size_t csize;
+  void* start = mi_os_page_align_area_conservative(addr, size, &csize);
+  if (csize == 0) return true;  // || _mi_os_is_huge_reserved(addr)
+  mi_os_stat_counter_increase(reset, csize);
+  mi_os_stat_counter_increase(reset_calls, 1);
+
+  #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN
+  memset(start, 0, csize); // pretend it is eagerly reset
+  #endif
+
+  int err = _mi_prim_reset(start, csize);
+  if (err != 0) {
+    _mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
+  }
+  return (err == 0);
+}
+
+
+void _mi_os_reuse( void* addr, size_t size ) {
+  // page align conservatively within the range
+  size_t csize = 0;
+  void* const start = mi_os_page_align_area_conservative(addr, size, &csize);
+  if (csize == 0) return;
+  const int err = _mi_prim_reuse(start, csize);
+  if (err != 0) {
+    _mi_warning_message("cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
+  }
+}
+
+// either resets or decommits memory, returns true if the memory needs
+// to be recommitted if it is to be re-used later on.
+bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)
+{
+  if (mi_option_get(mi_option_purge_delay) < 0) return false;  // is purging allowed?
+  mi_os_stat_counter_increase(purge_calls, 1);
+  mi_os_stat_counter_increase(purged, size);
+
+  if (commit_fun != NULL) {
+    bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);
+    return decommitted; // needs_recommit?
+  }
+  else if (mi_option_is_enabled(mi_option_purge_decommits) &&   // should decommit?
+           !_mi_preloading())                                   // don't decommit during preloading (unsafe)
+  {
+    bool needs_recommit = true;
+    mi_os_decommit_ex(p, size, &needs_recommit, stat_size);
+    return needs_recommit;
+  }
+  else {
+    if (allow_reset) {  // this can sometimes be not allowed if the range is not fully committed (on Windows, we cannot reset uncommitted memory)
+      _mi_os_reset(p, size);
+    }
+    return false;  // needs no recommit
+  }
+}
+
+// either resets or decommits memory, returns true if the memory needs
+// to be recommitted if it is to be re-used later on.
+bool _mi_os_purge(void* p, size_t size) {
+  return _mi_os_purge_ex(p, size, true, size, NULL, NULL);
+}
+
+
+// Protect a region in memory to be not accessible.
+static  bool mi_os_protectx(void* addr, size_t size, bool protect) {
+  // page align conservatively within the range
+  size_t csize = 0;
+  void* start = mi_os_page_align_area_conservative(addr, size, &csize);
+  if (csize == 0) return false;
+  /*
+  if (_mi_os_is_huge_reserved(addr)) {
+	  _mi_warning_message("cannot mprotect memory allocated in huge OS pages\n");
+  }
+  */
+  int err = _mi_prim_protect(start,csize,protect);
+  if (err != 0) {
+    _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize);
+  }
+  return (err == 0);
+}
+
+bool _mi_os_protect(void* addr, size_t size) {
+  return mi_os_protectx(addr, size, true);
+}
+
+bool _mi_os_unprotect(void* addr, size_t size) {
+  return mi_os_protectx(addr, size, false);
+}
+
+
+
+/* ----------------------------------------------------------------------------
+Support for allocating huge OS pages (1Gib) that are reserved up-front
+and possibly associated with a specific NUMA node. (use `numa_node>=0`)
+-----------------------------------------------------------------------------*/
+#define MI_HUGE_OS_PAGE_SIZE  (MI_GiB)
+
+
+#if (MI_INTPTR_SIZE >= 8)
+// To ensure proper alignment, use our own area for huge OS pages
+static mi_decl_cache_align _Atomic(uintptr_t)  mi_huge_start; // = 0
+
+// Claim an aligned address range for huge pages
+static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
+  if (total_size != NULL) *total_size = 0;
+  const size_t size = pages * MI_HUGE_OS_PAGE_SIZE;
+
+  uintptr_t start = 0;
+  uintptr_t end = 0;
+  uintptr_t huge_start = mi_atomic_load_relaxed(&mi_huge_start);
+  do {
+    start = huge_start;
+    if (start == 0) {
+      // Initialize the start address after the 32TiB area
+      start = ((uintptr_t)8 << 40);   // 8TiB virtual start address
+      #if (MI_SECURE>0 || MI_DEBUG==0)  // security: randomize start of huge pages unless in debug mode
+      mi_theap_t* const theap = _mi_theap_default();     // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
+      if (mi_theap_is_initialized(theap)) {              // todo: or no hint at all if we lack randomness?
+        const uintptr_t r = _mi_theap_random_next(theap);
+        start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF));  // (randomly 12bits)*1GiB == between 0 to 4TiB
+      }
+      else {
+        _mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, start);
+      }
+      #endif
+    }
+    end = start + size;
+  } while (!mi_atomic_cas_weak_acq_rel(&mi_huge_start, &huge_start, end));
+
+  if (total_size != NULL) *total_size = size;
+  return (uint8_t*)start;
+}
+#else
+static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
+  MI_UNUSED(pages);
+  if (total_size != NULL) *total_size = 0;
+  return NULL;
+}
+#endif
+
+// Allocate MI_ARENA_SLICE_ALIGN aligned huge pages
+void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {
+  *memid = _mi_memid_none();
+  if (psize != NULL) *psize = 0;
+  if (pages_reserved != NULL) *pages_reserved = 0;
+  size_t size = 0;
+  uint8_t* const start = mi_os_claim_huge_pages(pages, &size);
+  if (start == NULL) return NULL; // or 32-bit systems
+
+  // Allocate one page at the time but try to place them contiguously
+  // We allocate one page at the time to be able to abort if it takes too long
+  // or to at least allocate as many as available on the system.
+  mi_msecs_t start_t = _mi_clock_start();
+  size_t page = 0;
+  bool all_zero = true;
+  while (page < pages) {
+    // allocate a page
+    bool is_zero = false;
+    void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE);
+    void* p = NULL;
+    int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p);
+    if (!is_zero) { all_zero = false;  }
+    if (err != 0) {
+      _mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE);
+      break;
+    }
+
+    // Did we succeed at a contiguous address?
+    if (p != addr) {
+      // no success, issue a warning and break
+      if (p != NULL) {
+        _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
+        mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, NULL);
+      }
+      break;
+    }
+
+    // success, record it
+    page++;  // increase before timeout check (see issue #711)
+    mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);
+    mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);
+
+    // check for timeout
+    if (max_msecs > 0) {
+      mi_msecs_t elapsed = _mi_clock_end(start_t);
+      if (page >= 1) {
+        mi_msecs_t estimate = ((elapsed / (page+1)) * pages);
+        if (estimate > 2*max_msecs) { // seems like we are going to timeout, break
+          elapsed = max_msecs + 1;
+        }
+      }
+      if (elapsed > max_msecs) {
+        _mi_warning_message("huge OS page allocation timed out (after allocating %zu page(s))\n", page);
+        break;
+      }
+    }
+  }
+  const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE;
+  mi_assert_internal(allocated <= size);
+  if (pages_reserved != NULL) { *pages_reserved = page; }
+  if (psize != NULL) { *psize = allocated; }
+  if (page != 0) {
+    mi_assert(start != NULL);
+    *memid = _mi_memid_create_os(start, allocated, true /* is committed */, all_zero, true /* is_large */);
+    memid->memkind = MI_MEM_OS_HUGE;
+    mi_assert(memid->is_pinned);
+    #ifdef MI_TRACK_ASAN
+    if (all_zero) { mi_track_mem_defined(start,allocated); }
+    #endif
+  }
+  return (page == 0 ? NULL : start);
+}
+
+// free every huge page in a range individually (as we allocated per page)
+// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.
+static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc) {
+  if (p==NULL || size==0) return;
+  uint8_t* base = (uint8_t*)p;
+  while (size >= MI_HUGE_OS_PAGE_SIZE) {
+    mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, subproc);
+    size -= MI_HUGE_OS_PAGE_SIZE;
+    base += MI_HUGE_OS_PAGE_SIZE;
+  }
+}
+
+
+/* ----------------------------------------------------------------------------
+Support NUMA aware allocation
+-----------------------------------------------------------------------------*/
+
+static _Atomic(size_t) mi_numa_node_count; // = 0   // cache the node count
+
+int _mi_os_numa_node_count(void) {
+  size_t count = mi_atomic_load_acquire(&mi_numa_node_count);
+  if mi_unlikely(count == 0) {
+    long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly?
+    if (ncount > 0 && ncount < INT_MAX) {
+      count = (size_t)ncount;
+    }
+    else {
+      const size_t n = _mi_prim_numa_node_count(); // or detect dynamically
+      if (n == 0 || n > INT_MAX) { count = 1; }
+                            else { count = n; }
+    }
+    mi_atomic_store_release(&mi_numa_node_count, count); // save it
+    if (count>1) { _mi_verbose_message("using %zd numa regions\n", count); }
+  }
+  mi_assert_internal(count > 0 && count <= INT_MAX);
+  return (int)count;
+}
+
+static int mi_os_numa_node_get(void) {
+  int numa_count = _mi_os_numa_node_count();
+  if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0
+  // never more than the node count and >= 0
+  const size_t n = _mi_prim_numa_node();
+  int numa_node = (n < INT_MAX ? (int)n : 0);
+  if (numa_node >= numa_count) { numa_node = numa_node % numa_count; }
+  return numa_node;
+}
+
+int _mi_os_numa_node(void) {
+  if mi_likely(mi_atomic_load_relaxed(&mi_numa_node_count) == 1) {
+    return 0;
+  }
+  else {
+    return mi_os_numa_node_get();
+  }
+}
+
+
+/* ----------------------------------------------------------------------------
+  Public API
+-----------------------------------------------------------------------------*/
+#if 0
+mi_decl_export void* mi_os_alloc(size_t size, bool commit, size_t* full_size) {
+  return mi_os_alloc_aligned(size, mi_os_mem_config.alloc_granularity, commit, NULL, full_size);
+}
+
+static void* mi_os_alloc_aligned_ex(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) {
+  mi_memid_t memid = _mi_memid_none();
+  void* p = _mi_os_alloc_aligned(size, alignment, commit, allow_large, &memid);
+  if (p == NULL) return p;
+  if (is_committed != NULL) { *is_committed = memid.initially_committed;  }
+  if (is_pinned != NULL) { *is_pinned = memid.is_pinned;  }
+  if (base != NULL) { *base = memid.mem.os.base;  }
+  if (full_size != NULL) { *full_size = memid.mem.os.size;  }
+  if (!memid.initially_zero && memid.initially_committed) {
+    _mi_memzero_aligned(memid.mem.os.base, memid.mem.os.size);
+  }
+  return p;
+}
+
+mi_decl_export void* mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, void** base, size_t* full_size) {
+  return mi_os_alloc_aligned_ex(size, alignment, commit, false, NULL, NULL, base, full_size);
+}
+
+mi_decl_export void* mi_os_alloc_aligned_allow_large(size_t size, size_t alignment, bool commit, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) {
+  return mi_os_alloc_aligned_ex(size, alignment, commit, true, is_committed, is_pinned, base, full_size);
+}
+
+mi_decl_export void  mi_os_free(void* p, size_t size) {
+  if (p==NULL || size == 0) return;
+  mi_memid_t memid = _mi_memid_create_os(p, size, true, false, false);
+  _mi_os_free(p, size, memid);
+}
+
+mi_decl_export void  mi_os_commit(void* p, size_t size) {
+  _mi_os_commit(p, size, NULL);
+}
+
+mi_decl_export void  mi_os_decommit(void* p, size_t size) {
+  _mi_os_decommit(p, size);
+}
+#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-map.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-map.c
new file mode 100644
index 00000000..acc0ee9f
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-map.c
@@ -0,0 +1,447 @@
+/*----------------------------------------------------------------------------
+Copyright (c) 2023-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "bitmap.h"
+
+static void mi_page_map_cannot_commit(void) {
+  _mi_warning_message("unable to commit the allocation page-map on-demand\n" );
+}
+
+#if MI_PAGE_MAP_FLAT
+
+// The page-map contains a byte for each 64kb slice in the address space.
+// For an address `a` where `ofs = _mi_page_map[a >> 16]`:
+// 0 = unused
+// 1 = the slice at `a & ~0xFFFF` is a mimalloc page.
+// 1 < ofs <= 127 = the slice is part of a page, starting at `(((a>>16) - ofs - 1) << 16)`.
+//
+// 1 byte per slice => 1 TiB address space needs a 2^14 * 2^16 = 16 MiB page map.
+// A full 256 TiB address space (48 bit) needs a 4 GiB page map.
+// A full 4 GiB address space (32 bit) needs only a 64 KiB page map.
+
+mi_decl_cache_align uint8_t* _mi_page_map = NULL;
+static void*        mi_page_map_max_address = NULL;
+static mi_memid_t   mi_page_map_memid;
+
+#define MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT   MI_ARENA_SLICE_SIZE
+static mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entries
+
+mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count);
+
+bool _mi_page_map_init(void) {
+  size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);
+  if (vbits == 0) {
+    vbits = _mi_os_virtual_address_bits();
+    #if MI_ARCH_X64  // canonical address is limited to the first 128 TiB
+    if (vbits >= 48) { vbits = 47; }
+    #endif
+  }
+  if (vbits < MI_ARENA_SLICE_SHIFT) {
+    vbits = MI_ARENA_SLICE_SHIFT;
+  }
+
+  // Allocate the page map and commit bits
+  mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
+  const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT));
+  const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems?
+  const size_t commit_bits = _mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);
+  const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL));
+  const size_t reserve_size = bitmap_size + page_map_size;
+  uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);
+  if (base==NULL) {
+    _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);
+    return false;
+  }
+  if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {
+    _mi_warning_message("internal: the page map was committed but not zero initialized!\n");
+    _mi_memzero_aligned(base, reserve_size);
+  }
+  if (bitmap_size > 0) {
+    mi_page_map_commit = (mi_bitmap_t*)base;
+    if (!_mi_os_commit(mi_page_map_commit, bitmap_size, NULL)) {
+      mi_page_map_cannot_commit();
+      return false;
+    }
+    mi_bitmap_init(mi_page_map_commit, commit_bits, true);
+  }
+  _mi_page_map = base + bitmap_size;
+
+  // commit the first part so NULL pointers get resolved without an access violation
+  if (!commit) {
+    if (!mi_page_map_ensure_committed(0, 1)) {
+      mi_page_map_cannot_commit();
+      return false;
+    }
+  }
+  _mi_page_map[0] = 1; // so _mi_ptr_page(NULL) == NULL
+  mi_assert_internal(_mi_ptr_page(NULL)==NULL);
+  return true;
+}
+
+void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {
+  mi_assert_internal(subproc != NULL);
+  mi_assert_internal(_mi_page_map != NULL);
+  if (_mi_page_map == NULL) return;
+  _mi_os_free_ex(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);
+  _mi_page_map = NULL;
+  mi_page_map_commit = NULL;
+  mi_page_map_max_address = NULL;
+  mi_page_map_memid = _mi_memid_none();
+}
+
+
+static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {
+  // is the page map area that contains the page address committed?
+  // we always set the commit bits so we can track what ranges are in-use.
+  // we only actually commit if the map wasn't committed fully already.
+  if (mi_page_map_commit != NULL) {
+    const size_t commit_idx = idx / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
+    const size_t commit_idx_hi = (idx + slice_count - 1) / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
+    for (size_t i = commit_idx; i <= commit_idx_hi; i++) {  // per bit to avoid crossing over bitmap chunks
+      if (mi_bitmap_is_clear(mi_page_map_commit, i)) {
+        // this may race, in which case we do multiple commits (which is ok)
+        bool is_zero;
+        uint8_t* const start = _mi_page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);
+        const size_t   size  = MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
+        if (!_mi_os_commit(start, size, &is_zero)) {
+          mi_page_map_cannot_commit();
+          return false;
+        }
+        if (!is_zero && !mi_page_map_memid.initially_zero) { _mi_memzero(start, size); }
+        mi_bitmap_set(mi_page_map_commit, i);
+      }
+    }
+  }
+  #if MI_DEBUG > 0
+  _mi_page_map[idx] = 0;
+  _mi_page_map[idx+slice_count-1] = 0;
+  #endif
+  return true;
+}
+
+
+static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* slice_count) {
+  size_t page_size;
+  *page_start = mi_page_area(page, &page_size);
+  if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; }  // furthest interior pointer
+  *slice_count = mi_slice_count_of_size(page_size) + ((*page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks
+  return _mi_page_map_index(page);
+}
+
+bool _mi_page_map_register(mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_page_map != NULL);  // should be initialized before multi-thread access!
+  if mi_unlikely(_mi_page_map == NULL) {
+    if (!_mi_page_map_init()) return false;
+  }
+  mi_assert(_mi_page_map!=NULL);
+  uint8_t* page_start;
+  size_t   slice_count;
+  const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);
+
+  if (!mi_page_map_ensure_committed(idx, slice_count)) {
+    return false;
+  }
+
+  // set the offsets
+  for (size_t i = 0; i < slice_count; i++) {
+    mi_assert_internal(i < 128);
+    _mi_page_map[idx + i] = (uint8_t)(i+1);
+  }
+  return true;
+}
+
+void _mi_page_map_unregister(mi_page_t* page) {
+  mi_assert_internal(_mi_page_map != NULL);
+  // get index and count
+  uint8_t* page_start;
+  size_t   slice_count;
+  const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);
+  // unset the offsets
+  _mi_memzero(_mi_page_map + idx, slice_count);
+}
+
+void _mi_page_map_unregister_range(void* start, size_t size) {
+  const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
+  const uintptr_t index = _mi_page_map_index(start);
+  // todo: scan the commit bits and clear only those ranges?
+  if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total;
+    return;
+  }
+  _mi_memzero(&_mi_page_map[index], slice_count);
+}
+
+
+mi_page_t* _mi_safe_ptr_page(const void* p) {
+  if mi_unlikely(p >= mi_page_map_max_address) return NULL;
+  const uintptr_t idx = _mi_page_map_index(p);
+  if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL;
+  const uintptr_t ofs = _mi_page_map[idx];
+  if mi_unlikely(ofs == 0) return NULL;
+  return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT);
+}
+
+mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
+  return (_mi_safe_ptr_page(p) != NULL);
+}
+
+#else
+
+// A 2-level page map
+#define MI_PAGE_MAP_SUB_SIZE          (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*))
+#define MI_PAGE_MAP_ENTRIES_PER_CBIT  (MI_PAGE_MAP_COUNT < MI_BFIELD_BITS ? 1 : (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS))
+
+mi_decl_cache_align _Atomic(mi_submap_t)* _mi_page_map;
+static size_t       mi_page_map_count;
+static void*        mi_page_map_max_address;
+static mi_memid_t   mi_page_map_memid;
+static mi_lock_t    mi_page_map_lock;
+
+// divide the main map in 64 (`MI_BFIELD_BITS`) parts commit those parts on demand
+static _Atomic(mi_bfield_t)  mi_page_map_commit;
+
+mi_decl_nodiscard static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) {
+  mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit);
+  const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT;
+  mi_assert_internal(bit_idx < MI_BFIELD_BITS);
+  if (pbit_idx != NULL) { *pbit_idx = bit_idx; }
+  return ((commit & (MI_ZU(1) << bit_idx)) != 0);
+}
+
+mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, mi_submap_t* submap) {
+  mi_assert_internal(submap!=NULL && *submap==NULL);
+  size_t bit_idx;
+  if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) {
+    uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT];
+    if (!_mi_os_commit(start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_submap_t), NULL)) {
+      mi_page_map_cannot_commit();
+      return false;
+    }
+    mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx);
+  }
+  *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // acquire _mi_page_map_at(idx);
+  return true;
+}
+
+// initialize the page map
+bool _mi_page_map_init(void) {
+  size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);
+  if (vbits == 0) {
+    vbits = _mi_os_virtual_address_bits();
+    #if MI_ARCH_X64  // canonical address is limited to the first 128 TiB
+    if (vbits >= 48) { vbits = 47; }
+    #endif
+  }
+  if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) {
+    vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT;
+  }
+
+  // Allocate the page map and commit bits
+  mi_assert(MI_MAX_VABITS >= vbits);
+  mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
+  mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
+  mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT);
+  const size_t os_page_size = _mi_os_page_size();
+  const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size);
+  const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
+  const size_t reserve_size = page_map_size + submap_size;
+  #if MI_SECURE
+  const bool commit = true;  // the whole page map is valid and we can reliably check any pointer
+  #else
+  const bool commit = page_map_size <= 64*MI_KiB ||
+                      mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit();
+  #endif
+  _mi_page_map = (_Atomic(mi_page_t**)*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);
+  if (_mi_page_map==NULL) {
+    _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);
+    return false;
+  }
+  if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {
+    _mi_warning_message("internal: the page map was committed but not zero initialized!\n");
+    _mi_memzero_aligned(_mi_page_map, page_map_size);
+  }
+  mi_atomic_store_release(&mi_page_map_commit, (mi_page_map_memid.initially_committed ? ~MI_ZU(0) : MI_ZU(0)));
+
+  // ensure there is a submap for the NULL address
+  mi_page_t** const sub0 = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size);  // we reserved a submap part at the end already
+  if (!mi_page_map_memid.initially_committed) {
+    if (!_mi_os_commit(sub0, submap_size, NULL)) {  // commit full submap (issue #1087)
+      mi_page_map_cannot_commit();
+      return false;
+    }
+  }
+  if (!mi_page_map_memid.initially_zero) {     // initialize low addresses with NULL
+    _mi_memzero_aligned(sub0, submap_size);
+  }
+  mi_submap_t nullsub = NULL;
+  if (!mi_page_map_ensure_committed(0,&nullsub)) {
+    mi_page_map_cannot_commit();
+    return false;
+  }
+  mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[0], sub0);
+  mi_lock_init(&mi_page_map_lock);             // initialize late in case the lock init causes allocation
+  
+  mi_assert_internal(_mi_ptr_page(NULL)==NULL);
+  return true;
+}
+
+
+void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {
+  mi_assert_internal(subproc != NULL);
+  mi_assert_internal(_mi_page_map != NULL);
+  if (_mi_page_map == NULL) return;
+  mi_lock_done(&mi_page_map_lock);  
+  for (size_t idx = 1; idx < mi_page_map_count; idx++) {  // skip entry 0 (as we allocate that submap at the end of the page_map)
+    // free all sub-maps
+    if (mi_page_map_is_committed(idx, NULL)) {
+      mi_submap_t sub = _mi_page_map_at(idx);
+      if (sub != NULL) {
+        mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false);
+        _mi_os_free_ex(memid.mem.os.base, memid.mem.os.size, true, memid, subproc);
+        mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[idx], NULL);
+      }
+    }
+  }
+  _mi_os_free_ex(_mi_page_map, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);
+  _mi_page_map = NULL;
+  mi_page_map_count = 0;
+  mi_page_map_memid = _mi_memid_none();
+  mi_page_map_max_address = NULL;
+  mi_atomic_store_release(&mi_page_map_commit, (mi_bfield_t)0);
+}
+
+
+mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap_t* submap) {
+  mi_assert_internal(submap!=NULL && *submap==NULL);
+  mi_submap_t sub = NULL;
+  if (!mi_page_map_ensure_committed(idx, &sub)) {
+    return false;
+  }
+  if mi_unlikely(sub == NULL) {
+    // sub map not yet allocated, alloc now
+    mi_lock(&mi_page_map_lock) 
+    {
+      sub = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // reload
+      if (sub==NULL) // not yet allocated by another thread?      
+      {
+        mi_memid_t memid;
+        const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
+        sub = (mi_submap_t)_mi_os_zalloc(submap_size, &memid);        
+        if (sub==NULL) {
+          _mi_warning_message("internal error: unable to extend the page map\n");          
+        }
+        else {
+          mi_submap_t expect = NULL;
+          if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &_mi_page_map[idx], &expect, sub)) {
+            // another thread already allocated it.. free and continue
+            _mi_os_free(sub, submap_size, memid);
+            sub = expect;
+          }
+        }
+      }
+    }
+    if (sub==NULL) return false; // unable to allocate the submap..
+  }
+  mi_assert_internal(sub!=NULL);
+  *submap = sub;
+  return true;
+}
+
+static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
+  // is the page map area that contains the page address committed?
+  while (slice_count > 0) {
+    mi_submap_t sub = NULL;
+    if (!mi_page_map_ensure_submap_at(idx, &sub)) {
+      return false;
+    };
+    mi_assert_internal(sub!=NULL);
+    // set the offsets for the page
+    while (slice_count > 0 && sub_idx < MI_PAGE_MAP_SUB_COUNT) {
+      sub[sub_idx] = page;
+      slice_count--;
+      sub_idx++;
+    }
+    idx++; // potentially wrap around to the next idx
+    sub_idx = 0;
+  }
+  return true;
+}
+
+static bool mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
+  if mi_unlikely(!mi_page_map_set_range_prim(page,idx,sub_idx,slice_count)) {
+    // failed to commit, call again to reset the page pointer if needed
+    if (page!=NULL) {
+      mi_page_map_set_range_prim(NULL,idx,sub_idx,slice_count);
+    }
+    return false;
+  }
+  return true;
+}
+
+static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slice_count) {
+  size_t page_size;
+  uint8_t* page_start = mi_page_area(page, &page_size);
+  if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; }  // furthest interior pointer
+  *slice_count = mi_slice_count_of_size(page_size) + ((page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks
+  return _mi_page_map_index(page_start, sub_idx);
+}
+
+bool _mi_page_map_register(mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_page_map != NULL);  // should be initialized before multi-thread access!
+  if mi_unlikely(_mi_page_map == NULL) {
+    if (!_mi_page_map_init()) return false;
+  }
+  mi_assert(_mi_page_map!=NULL);
+  size_t   slice_count;
+  size_t   sub_idx;
+  const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
+  return mi_page_map_set_range(page, idx, sub_idx, slice_count);
+}
+
+void _mi_page_map_unregister(mi_page_t* page) {
+  mi_assert_internal(_mi_page_map != NULL);
+  mi_assert_internal(page != NULL);
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  if mi_unlikely(_mi_page_map == NULL) return;
+  // get index and count
+  size_t slice_count;
+  size_t sub_idx;
+  const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
+  // unset the offsets
+  mi_page_map_set_range(NULL, idx, sub_idx, slice_count);
+}
+
+void _mi_page_map_unregister_range(void* start, size_t size) {
+  if mi_unlikely(_mi_page_map == NULL) return;
+  const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
+  size_t sub_idx;
+  const uintptr_t idx = _mi_page_map_index(start, &sub_idx);
+  mi_page_map_set_range(NULL, idx, sub_idx, slice_count);  // todo: avoid committing if not already committed?
+}
+
+// Return NULL for invalid pointers
+mi_page_t* _mi_safe_ptr_page(const void* p) {
+  if (p==NULL) return NULL;
+  if mi_unlikely(p >= mi_page_map_max_address) return NULL;
+  size_t sub_idx;
+  const size_t idx = _mi_page_map_index(p,&sub_idx);
+  if mi_unlikely(!mi_page_map_is_committed(idx,NULL)) return NULL;
+  mi_page_t** const sub = _mi_page_map[idx];
+  if mi_unlikely(sub==NULL) return NULL;
+  return sub[sub_idx];
+}
+
+mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
+  return (_mi_safe_ptr_page(p) != NULL);
+}
+
+#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-queue.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-queue.c
new file mode 100644
index 00000000..731672b0
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-queue.c
@@ -0,0 +1,458 @@
+/*----------------------------------------------------------------------------
+Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+/* -----------------------------------------------------------
+  Definition of page queues for each block size
+----------------------------------------------------------- */
+
+#ifndef MI_IN_PAGE_C
+#error "this file should be included from 'page.c'"
+// include to help an IDE
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/atomic.h"
+#endif
+
+/* -----------------------------------------------------------
+  Minimal alignment in machine words (i.e. `sizeof(void*)`)
+----------------------------------------------------------- */
+
+#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE)
+  #error "define alignment for more than 4x word size for this platform"
+#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE)
+  #define MI_ALIGN4W   // 4 machine words minimal alignment
+#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE)
+  #define MI_ALIGN2W   // 2 machine words minimal alignment
+#else
+  // ok, default alignment is 1 word
+#endif
+
+
+/* -----------------------------------------------------------
+  Queue query
+----------------------------------------------------------- */
+
+
+static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) {
+  return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+sizeof(uintptr_t)));
+}
+
+static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) {
+  return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+(2*sizeof(uintptr_t))));
+}
+
+static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) {
+  return (pq->block_size > MI_LARGE_MAX_OBJ_SIZE);
+}
+
+static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) {
+  return pq->count;
+}
+
+/* -----------------------------------------------------------
+  Bins
+----------------------------------------------------------- */
+
+// Return the bin for a given field size.
+// Returns MI_BIN_HUGE if the size is too large.
+// We use `wsize` for the size in "machine word sizes",
+// i.e. byte size == `wsize*sizeof(void*)`.
+static mi_decl_noinline size_t mi_bin(size_t size) {
+  size_t wsize = _mi_wsize_from_size(size);
+#if defined(MI_ALIGN4W)
+  if mi_likely(wsize <= 4) {
+    return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
+  }
+#elif defined(MI_ALIGN2W)
+  if mi_likely(wsize <= 8) {
+    return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
+  }
+#else
+  if mi_likely(wsize <= 8) {
+    return (wsize == 0 ? 1 : wsize);
+  }
+#endif
+  else if mi_unlikely(wsize > MI_LARGE_MAX_OBJ_WSIZE) {
+    return MI_BIN_HUGE;
+  }
+  else {
+    #if defined(MI_ALIGN4W)
+    if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes
+    #endif
+    wsize--;
+    // find the highest bit
+    const size_t b = (MI_SIZE_BITS - 1 - mi_clz(wsize));  // note: wsize != 0
+    // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).
+    // - adjust with 3 because we use do not round the first 8 sizes
+    //   which each get an exact bin
+    const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3;
+    mi_assert_internal(bin > 0 && bin < MI_BIN_HUGE);
+    return bin;
+  }
+}
+
+
+
+/* -----------------------------------------------------------
+  Queue of pages with free blocks
+----------------------------------------------------------- */
+
+size_t _mi_bin(size_t size) {
+  return mi_bin(size);
+}
+
+size_t _mi_bin_size(size_t bin) {
+  mi_assert_internal(bin <= MI_BIN_HUGE);
+  return _mi_theap_empty.pages[bin].block_size;
+}
+
+// Good size for allocation
+mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept {
+  if (size <= MI_LARGE_MAX_OBJ_SIZE) {
+    return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE));
+  }
+  else if (size <= MI_MAX_ALLOC_SIZE) {
+    return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size());
+  }
+  else {
+    return size;
+  }
+}
+
+#if (MI_DEBUG>1)
+static bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  mi_page_t* list = queue->first;
+  while (list != NULL) {
+    mi_assert_internal(list->next == NULL || list->next->prev == list);
+    mi_assert_internal(list->prev == NULL || list->prev->next == list);
+    if (list == page) break;
+    list = list->next;
+  }
+  return (list == page);
+}
+
+#endif
+
+#if (MI_DEBUG>1)
+static bool mi_theap_contains_queue(const mi_theap_t* theap, const mi_page_queue_t* pq) {
+  return (pq >= &theap->pages[0] && pq <= &theap->pages[MI_BIN_FULL]);
+}
+#endif
+
+bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) {
+  MI_UNUSED_RELEASE(theap);
+  if (pq==NULL) return false;
+  size_t count = 0; MI_UNUSED_RELEASE(count);
+  mi_page_t* prev = NULL; MI_UNUSED_RELEASE(prev);
+  for (mi_page_t* page = pq->first; page != NULL; page = page->next) {
+    mi_assert_internal(page->prev == prev);
+    if (mi_page_is_in_full(page)) {
+      mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2);
+    }
+    else if (mi_page_is_huge(page)) {
+      mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1);
+    }
+    else {
+      mi_assert_internal(mi_page_block_size(page) == pq->block_size);
+    }
+    mi_assert_internal(page->theap == theap);
+    if (page->next == NULL) {
+      mi_assert_internal(pq->last == page);
+    }
+    count++;
+    prev = page;
+  }
+  mi_assert_internal(pq->count == count);
+  return true;
+}
+
+static size_t mi_page_bin(const mi_page_t* page) {
+  const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))));
+  mi_assert_internal(bin <= MI_BIN_FULL);
+  return bin;
+}
+
+// returns the page bin without using MI_BIN_FULL for statistics
+size_t _mi_page_stats_bin(const mi_page_t* page) {
+  const size_t bin = (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)));
+  mi_assert_internal(bin <= MI_BIN_HUGE);
+  return bin;
+}
+
+static mi_page_queue_t* mi_theap_page_queue_of(mi_theap_t* theap, const mi_page_t* page) {
+  mi_assert_internal(theap!=NULL);
+  const size_t bin = mi_page_bin(page);
+  mi_page_queue_t* pq = &theap->pages[bin];
+  mi_assert_internal((mi_page_block_size(page) == pq->block_size) ||
+                       (mi_page_is_huge(page) && mi_page_queue_is_huge(pq)) ||
+                         (mi_page_is_in_full(page) && mi_page_queue_is_full(pq)));
+  return pq;
+}
+
+static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) {
+  mi_theap_t* theap = mi_page_theap(page);
+  mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
+  mi_assert_expensive(mi_page_queue_contains(pq, page));
+  return pq;
+}
+
+// The current small page array is for efficiency and for each
+// small size (up to 256) it points directly to the page for that
+// size without having to compute the bin. This means when the
+// current free page queue is updated for a small bin, we need to update a
+// range of entries in `_mi_page_small_free`.
+static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_queue_t* pq) {
+  mi_assert_internal(mi_theap_contains_queue(theap,pq));
+  const size_t size = pq->block_size;
+  if (size > MI_SMALL_SIZE_MAX) return;
+
+  mi_page_t* page = pq->first;
+  if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty;
+
+  // find index in the right direct page array
+  const size_t idx = _mi_wsize_from_size(size);
+  mi_page_t** const pages_free = theap->pages_free_direct;
+  if (pages_free[idx] == page) return;  // already set
+
+  // find start slot
+  size_t start;
+  if (idx<=1) {
+    start = 0;
+  }
+  else {
+    // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped
+    mi_assert_internal(pq > &theap->pages[0]); // since idx > 1    
+    size_t bin = mi_bin(size);
+    const mi_page_queue_t* prev = pq - 1;
+    while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) {
+      prev--;
+    }
+    start = 1 + _mi_wsize_from_size(prev->block_size);
+    if (start > idx) start = idx;
+  }
+
+  // set size range to the right page
+  mi_assert(start <= idx);
+  for (size_t sz = start; sz <= idx; sz++) {
+    pages_free[sz] = page;
+  }
+}
+
+/*
+static bool mi_page_queue_is_empty(mi_page_queue_t* queue) {
+  return (queue->first == NULL);
+}
+*/
+
+static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  mi_assert_expensive(mi_page_queue_contains(queue, page));
+  mi_assert_internal(queue->count >= 1);
+  mi_assert_internal(mi_page_block_size(page) == queue->block_size ||
+                      (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||
+                        (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
+  mi_theap_t* theap = mi_page_theap(page);
+  if (page->prev != NULL) page->prev->next = page->next;
+  if (page->next != NULL) page->next->prev = page->prev;
+  if (page == queue->last)  queue->last = page->prev;
+  if (page == queue->first) {
+    queue->first = page->next;
+    // update first
+    mi_assert_internal(mi_theap_contains_queue(theap, queue));
+    mi_theap_queue_first_update(theap,queue);
+  }
+  theap->page_count--;
+  queue->count--;
+  page->next = NULL;
+  page->prev = NULL;
+  mi_page_set_in_full(page,false);
+}
+
+
+static void mi_page_queue_push(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
+  mi_assert_internal(mi_page_theap(page) == theap);
+  mi_assert_internal(!mi_page_queue_contains(queue, page));
+  #if MI_HUGE_PAGE_ABANDON
+  mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE);
+  #endif
+  mi_assert_internal(mi_page_block_size(page) == queue->block_size ||
+                      (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||
+                        (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
+
+  mi_page_set_in_full(page, mi_page_queue_is_full(queue));
+
+  page->next = queue->first;
+  page->prev = NULL;
+  if (queue->first != NULL) {
+    mi_assert_internal(queue->first->prev == NULL);
+    queue->first->prev = page;
+    queue->first = page;
+  }
+  else {
+    queue->first = queue->last = page;
+  }
+  queue->count++;
+
+  // update direct
+  mi_theap_queue_first_update(theap, queue);
+  theap->page_count++;
+}
+
+static void mi_page_queue_push_at_end(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
+  mi_assert_internal(mi_page_theap(page) == theap);
+  mi_assert_internal(!mi_page_queue_contains(queue, page));
+
+  mi_assert_internal(mi_page_block_size(page) == queue->block_size ||
+                      (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||
+                       (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
+
+  mi_page_set_in_full(page, mi_page_queue_is_full(queue));
+
+  page->prev = queue->last;
+  page->next = NULL;
+  if (queue->last != NULL) {
+    mi_assert_internal(queue->last->next == NULL);
+    queue->last->next = page;
+    queue->last = page;
+  }
+  else {
+    queue->first = queue->last = page;
+  }
+  queue->count++;
+
+  // update direct
+  if (queue->first == page) {
+    mi_theap_queue_first_update(theap, queue);
+  }
+  theap->page_count++;
+}
+
+static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
+  mi_assert_internal(mi_page_theap(page) == theap);
+  mi_assert_internal(mi_page_queue_contains(queue, page));
+  if (queue->first == page) return;
+  mi_page_queue_remove(queue, page);
+  mi_page_queue_push(theap, queue, page);
+  mi_assert_internal(queue->first == page);
+}
+
+static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  mi_assert_internal(from->count >= 1);
+  mi_assert_expensive(mi_page_queue_contains(from, page));
+  mi_assert_expensive(!mi_page_queue_contains(to, page));
+  const size_t bsize = mi_page_block_size(page);
+  MI_UNUSED(bsize);
+  mi_assert_internal((bsize == to->block_size && bsize == from->block_size) ||
+                     (bsize == to->block_size && mi_page_queue_is_full(from)) ||
+                     (bsize == from->block_size && mi_page_queue_is_full(to)) ||
+                     (mi_page_is_huge(page) && mi_page_queue_is_huge(to)) ||
+                     (mi_page_is_huge(page) && mi_page_queue_is_full(to)));
+
+  mi_theap_t* theap = mi_page_theap(page);
+
+  // delete from `from`
+  if (page->prev != NULL) page->prev->next = page->next;
+  if (page->next != NULL) page->next->prev = page->prev;
+  if (page == from->last)  from->last = page->prev;
+  if (page == from->first) {
+    from->first = page->next;
+    // update first
+    mi_assert_internal(mi_theap_contains_queue(theap, from));
+    mi_theap_queue_first_update(theap, from);
+  }
+  from->count--;
+
+  // insert into `to`
+  to->count++;
+  if (enqueue_at_end) {
+    // enqueue at the end
+    page->prev = to->last;
+    page->next = NULL;
+    if (to->last != NULL) {
+      mi_assert_internal(theap == mi_page_theap(to->last));
+      to->last->next = page;
+      to->last = page;
+    }
+    else {
+      to->first = page;
+      to->last = page;
+      mi_theap_queue_first_update(theap, to);
+    }
+  }
+  else {
+    if (to->first != NULL) {
+      // enqueue at 2nd place
+      mi_assert_internal(theap == mi_page_theap(to->first));
+      mi_page_t* next = to->first->next;
+      page->prev = to->first;
+      page->next = next;
+      to->first->next = page;
+      if (next != NULL) {
+        next->prev = page;
+      }
+      else {
+        to->last = page;
+      }
+    }
+    else {
+      // enqueue at the head (singleton list)
+      page->prev = NULL;
+      page->next = NULL;
+      to->first = page;
+      to->last = page;
+      mi_theap_queue_first_update(theap, to);
+    }
+  }
+
+  mi_page_set_in_full(page, mi_page_queue_is_full(to));
+}
+
+static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {
+  mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end */, page);
+}
+
+static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {
+  // note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`)
+  mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page);
+}
+
+// Only called from `mi_theap_absorb`.
+size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append) {
+  mi_assert_internal(mi_theap_contains_queue(theap,pq));
+  mi_assert_internal(pq->block_size == append->block_size);
+
+  if (append->first==NULL) return 0;
+
+  // set append pages to new theap and count
+  size_t count = 0;
+  for (mi_page_t* page = append->first; page != NULL; page = page->next) {
+    mi_page_set_theap(page, theap);
+    count++;
+  }
+  mi_assert_internal(count == append->count);
+
+  if (pq->last==NULL) {
+    // take over afresh
+    mi_assert_internal(pq->first==NULL);
+    pq->first = append->first;
+    pq->last = append->last;
+    mi_theap_queue_first_update(theap, pq);
+  }
+  else {
+    // append to end
+    mi_assert_internal(pq->last!=NULL);
+    mi_assert_internal(append->first!=NULL);
+    pq->last->next = append->first;
+    append->first->prev = pq->last;
+    pq->last = append->last;
+  }
+  pq->count += append->count;
+
+  return count;
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page.c
new file mode 100644
index 00000000..97d2fc27
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page.c
@@ -0,0 +1,1039 @@
+/*----------------------------------------------------------------------------
+Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+/* -----------------------------------------------------------
+  The core of the allocator. Every segment contains
+  pages of a certain block size. The main function
+  exported is `mi_malloc_generic`.
+----------------------------------------------------------- */
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/atomic.h"
+#include "mimalloc/prim.h"
+
+/* -----------------------------------------------------------
+  Definition of page queues for each block size
+----------------------------------------------------------- */
+
+#define MI_IN_PAGE_C
+#include "page-queue.c"
+#undef MI_IN_PAGE_C
+
+
+/* -----------------------------------------------------------
+  Page helpers
+----------------------------------------------------------- */
+
+// Index a block in a page
+static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) {
+  MI_UNUSED(page);
+  mi_assert_internal(page != NULL);
+  mi_assert_internal(i <= page->reserved);
+  return (mi_block_t*)((uint8_t*)page_start + (i * block_size));
+}
+
+static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page);
+
+#if (MI_DEBUG>=3)
+static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) {
+  mi_assert_internal(_mi_ptr_page(page->page_start) == page);
+  const uint8_t* slice_start = mi_page_slice_start(page);
+  mi_assert_internal(_mi_is_aligned(slice_start,MI_PAGE_ALIGN));
+  size_t count = 0;
+  while (head != NULL) {
+    mi_assert_internal((uint8_t*)head - slice_start > (ptrdiff_t)MI_LARGE_PAGE_SIZE || page == _mi_ptr_page(head));
+    count++;
+    head = mi_block_next(page, head);
+  }
+  return count;
+}
+
+/*
+// Start of the page available memory
+static inline uint8_t* mi_page_area(const mi_page_t* page) {
+  return _mi_page_start(_mi_page_segment(page), page, NULL);
+}
+*/
+
+static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) {
+  size_t psize;
+  uint8_t* page_area = mi_page_area(page, &psize);
+  mi_block_t* start = (mi_block_t*)page_area;
+  mi_block_t* end   = (mi_block_t*)(page_area + psize);
+  while(p != NULL) {
+    if (p < start || p >= end) return false;
+    p = mi_block_next(page, p);
+  }
+#if MI_DEBUG>3 // generally too expensive to check this
+  if (page->free_is_zero) {
+    const size_t ubsize = mi_page_usable_block_size(page);
+    for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {
+      mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));
+    }
+  }
+#endif
+  return true;
+}
+
+static bool mi_page_is_valid_init(mi_page_t* page) {
+  mi_assert_internal(mi_page_block_size(page) > 0);
+  mi_assert_internal(page->used <= page->capacity);
+  mi_assert_internal(page->capacity <= page->reserved);
+
+  // const size_t bsize = mi_page_block_size(page);
+  // uint8_t* start = mi_page_start(page);
+  //mi_assert_internal(start + page->capacity*page->block_size == page->top);
+
+  mi_assert_internal(mi_page_list_is_valid(page,page->free));
+  mi_assert_internal(mi_page_list_is_valid(page,page->local_free));
+
+  #if MI_DEBUG>3 // generally too expensive to check this
+  if (page->free_is_zero) {
+    const size_t ubsize = mi_page_usable_block_size(page);
+    for(mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) {
+      mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));
+    }
+  }
+  #endif
+
+  #if !MI_TRACK_ENABLED && !MI_TSAN
+  mi_block_t* tfree = mi_page_thread_free(page);
+  mi_assert_internal(mi_page_list_is_valid(page, tfree));
+  //size_t tfree_count = mi_page_list_count(page, tfree);
+  //mi_assert_internal(tfree_count <= page->thread_freed + 1);
+  #endif
+
+  size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free);
+  mi_assert_internal(page->used + free_count == page->capacity);
+
+  return true;
+}
+
+extern mi_decl_hidden bool _mi_process_is_initialized;             // has mi_process_init been called?
+
+bool _mi_page_is_valid(mi_page_t* page) {
+  mi_assert_internal(mi_page_is_valid_init(page));
+  #if MI_SECURE
+  mi_assert_internal(page->keys[0] != 0);
+  #endif
+  if (!mi_page_is_abandoned(page)) {
+    //mi_assert_internal(!_mi_process_is_initialized);
+    {
+      mi_page_queue_t* pq = mi_page_queue_of(page);
+      mi_assert_internal(mi_page_queue_contains(pq, page));
+      mi_assert_internal(pq->block_size==mi_page_block_size(page) || mi_page_is_huge(page) || mi_page_is_in_full(page));
+      // mi_assert_internal(mi_theap_contains_queue(mi_page_theap(page),pq));
+    }
+  }
+  return true;
+}
+#endif
+
+
+/* -----------------------------------------------------------
+  Page collect the `local_free` and `thread_free` lists
+----------------------------------------------------------- */
+
+static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head)
+{
+  if (head == NULL) return;
+
+  // find the last block in the list -- also to get a proper use count (without data races)
+  size_t max_count = page->capacity; // cannot collect more than capacity
+  size_t count = 1;
+  mi_block_t* last = head;
+  mi_block_t* next;
+  while ((next = mi_block_next(page, last)) != NULL && count <= max_count) {
+    count++;
+    last = next;
+  }
+
+  // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free)
+  if mi_unlikely(count > max_count) {
+    _mi_error_message(EFAULT, "corrupted thread-free list\n");
+    return; // the thread-free items cannot be freed
+  }
+  // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list)
+  else if mi_unlikely(count > page->used) {
+    _mi_error_message(EFAULT, "corrupted meta-data in thread-free list\n");
+    return; // the thread-free items cannot be freed
+  }
+
+  // and append the current local free list
+  mi_block_set_next(page, last, page->local_free);
+  page->local_free = head;
+
+  // update counts now
+  mi_assert_internal(count <= UINT16_MAX);
+  mi_assert_internal(page->used >= (uint16_t)count);
+  page->used = page->used - (uint16_t)count;
+}
+
+// Collect the local `thread_free` list using an atomic exchange.
+static void mi_page_thread_free_collect(mi_page_t* page)
+{
+  // atomically capture the thread free list
+  mi_block_t* head;
+  mi_thread_free_t tfreex;
+  mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free);
+  do {
+    head = mi_tf_block(tfree);
+    if mi_likely(head == NULL) return; // return if the list is empty
+    tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree));  // set the thread free list to NULL
+  } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex));  // release is enough?
+  mi_assert_internal(head != NULL);
+
+  // and move it to the local list
+  mi_page_thread_collect_to_local(page, head);
+}
+
+// returns `true` if after collection `mi_page_immediate_available` is true.
+static bool mi_page_free_quick_collect(mi_page_t* page) {
+  if (page->free != NULL) return true;
+  if (page->local_free == NULL) return false;
+  // move local_free to free
+  page->free = page->local_free;
+  page->local_free = NULL;
+  page->free_is_zero = false;
+  return true;
+}
+
+void _mi_page_free_collect(mi_page_t* page, bool force) {
+  mi_assert_internal(page!=NULL);
+
+  // collect the thread free list
+  mi_page_thread_free_collect(page);
+
+  // and the local free list
+  if (page->local_free != NULL) {
+    if mi_likely(page->free == NULL) {
+      // usual case
+      page->free = page->local_free;
+      page->local_free = NULL;
+      page->free_is_zero = false;
+    }
+    else if (force) {
+      // append -- only on shutdown (force) as this is a linear operation
+      mi_block_t* tail = page->local_free;
+      mi_block_t* next;
+      while ((next = mi_block_next(page, tail)) != NULL) {
+        tail = next;
+      }
+      mi_block_set_next(page, tail, page->free);
+      page->free = page->local_free;
+      page->local_free = NULL;
+      page->free_is_zero = false;
+    }
+  }
+
+  mi_assert_internal(!force || page->local_free == NULL);
+}
+
+// Collect elements in the thread-free list starting at `head`. This is an optimized
+// version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`.
+//
+// `head` must be in the `xthread_free` list. It will not collect `head` itself
+// so the `used` count is not fully updated in general. However, if the `head` is
+// the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true).
+void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {
+  if (head == NULL) return;
+  mi_block_t* next = mi_block_next(page,head);  // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops)
+  if (next != NULL) {
+    mi_block_set_next(page, head, NULL);
+    mi_page_thread_collect_to_local(page, next);
+    if (page->local_free != NULL && page->free == NULL) {
+      page->free = page->local_free;
+      page->local_free = NULL;
+      page->free_is_zero = false;
+    }
+  }
+  if (page->used == 1) {
+    // all elements are free'd since we skipped the `head` element itself
+    mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head);
+    mi_assert_internal(mi_block_next(page,head) == NULL);
+    _mi_page_free_collect(page, false);  // collect the final element
+  }
+}
+
+
+/* -----------------------------------------------------------
+  Page fresh and retire
+----------------------------------------------------------- */
+
+/*
+// called from segments when reclaiming abandoned pages
+void _mi_page_reclaim(mi_theap_t* theap, mi_page_t* page) {
+  // mi_page_set_theap(page, theap);
+  // _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after theap is set)
+  _mi_page_free_collect(page, false); // ensure used count is up to date
+
+  mi_assert_expensive(mi_page_is_valid_init(page));
+  // mi_assert_internal(mi_page_theap(page) == theap);
+  // mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE);
+
+  // TODO: push on full queue immediately if it is full?
+  mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
+  mi_page_queue_push(theap, pq, page);
+  mi_assert_expensive(_mi_page_is_valid(page));
+}
+*/
+
+// called from `mi_free` on a reclaim, and fresh_alloc if we get an abandoned page
+void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page)
+{
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(mi_page_is_abandoned(page));
+
+  mi_page_set_theap(page,theap);
+  _mi_page_free_collect(page, false); // ensure used count is up to date
+  mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
+  mi_page_queue_push_at_end(theap, pq, page);
+  mi_assert_expensive(_mi_page_is_valid(page));
+}
+
+void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
+  _mi_page_free_collect(page, false); // ensure used count is up to date
+  if (mi_page_all_free(page)) {
+    _mi_page_free(page, pq);
+  }
+  else {
+    mi_page_queue_remove(pq, page);
+    mi_theap_t* theap = page->theap;
+    mi_page_set_theap(page, NULL);
+    page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap
+    _mi_arenas_page_abandon(page, theap);
+    _mi_arenas_collect(false, false, theap->tld); // allow purging
+  }
+}
+
+
+// allocate a fresh page from an arena
+static mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, size_t block_size, size_t page_alignment) {
+  #if !MI_HUGE_PAGE_ABANDON
+  mi_assert_internal(pq != NULL);
+  mi_assert_internal(mi_theap_contains_queue(theap, pq));
+  mi_assert_internal(page_alignment > 0 || block_size > MI_LARGE_MAX_OBJ_SIZE || block_size == pq->block_size);
+  #endif
+  mi_page_t* page = _mi_arenas_page_alloc(theap, block_size, page_alignment);
+  if (page == NULL) {
+    // out-of-memory
+    return NULL;
+  }
+  if (mi_page_is_abandoned(page)) {
+    _mi_theap_page_reclaim(theap, page);
+    if (!mi_page_immediate_available(page)) {
+      if (mi_page_is_expandable(page)) {
+        if (!mi_page_extend_free(theap, page)) {
+          return NULL; // cannot commit
+        };
+      }
+      else {
+        mi_assert(false); // should not happen?
+        return NULL;
+      }
+    }
+  }
+  else if (pq != NULL) {
+    mi_page_queue_push(theap, pq, page);
+  }
+  mi_assert_internal(pq!=NULL || mi_page_block_size(page) >= block_size);
+  mi_assert_expensive(_mi_page_is_valid(page));
+  return page;
+}
+
+// Get a fresh page to use
+static mi_page_t* mi_page_fresh(mi_theap_t* theap, mi_page_queue_t* pq) {
+  mi_assert_internal(mi_theap_contains_queue(theap, pq));
+  mi_page_t* page = mi_page_fresh_alloc(theap, pq, pq->block_size, 0);
+  if (page==NULL) return NULL;
+  mi_assert_internal(pq->block_size==mi_page_block_size(page));
+  mi_assert_internal(pq==mi_theap_page_queue_of(theap, page));
+  return page;
+}
+
+
+/* -----------------------------------------------------------
+  Unfull, abandon, free and retire
+----------------------------------------------------------- */
+
+// Move a page from the full list back to a regular list (called from thread-local mi_free)
+void _mi_page_unfull(mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  mi_assert_expensive(_mi_page_is_valid(page));
+  mi_assert_internal(mi_page_is_in_full(page));
+  mi_assert_internal(!mi_page_theap(page)->allow_page_abandon);
+  if (!mi_page_is_in_full(page)) return;
+
+  mi_theap_t* theap = mi_page_theap(page);
+  mi_page_queue_t* pqfull = &theap->pages[MI_BIN_FULL];
+  mi_page_set_in_full(page, false); // to get the right queue
+  mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
+  mi_page_set_in_full(page, true);
+  mi_page_queue_enqueue_from_full(pq, pqfull, page);
+}
+
+static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
+  mi_assert_internal(pq == mi_page_queue_of(page));
+  mi_assert_internal(!mi_page_immediate_available(page));
+  mi_assert_internal(!mi_page_is_in_full(page));
+
+  mi_theap_t* theap = mi_page_theap(page);
+  if (theap->allow_page_abandon) {
+    // abandon full pages (this is the usual case in order to allow for sharing of memory between theaps)
+    _mi_page_abandon(page, pq);
+  }
+  else if (!mi_page_is_in_full(page)) {
+    // put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed)
+    mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page);
+    _mi_page_free_collect(page, false);  // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set
+  }
+}
+
+
+// Free a page with no more free blocks
+void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) {
+  mi_assert_internal(page != NULL);
+  mi_assert_expensive(_mi_page_is_valid(page));
+  mi_assert_internal(pq == mi_page_queue_of(page));
+  mi_assert_internal(mi_page_all_free(page));
+  // mi_assert_internal(mi_page_thread_free_flag(page)!=MI_DELAYED_FREEING);
+
+  // no more aligned blocks in here
+  mi_page_set_has_interior_pointers(page, false);
+
+  // remove from the page list
+  // (no need to do _mi_theap_delayed_free first as all blocks are already free)
+  mi_page_queue_remove(pq, page);
+
+  // and free it
+  mi_theap_t* theap = mi_page_theap(page); mi_assert_internal(theap!=NULL);
+  mi_page_set_theap(page,NULL);
+  _mi_arenas_page_free(page, theap);
+  _mi_arenas_collect(false, false, theap->tld);  // allow purging
+}
+
+#define MI_MAX_RETIRE_SIZE    MI_LARGE_OBJ_SIZE_MAX   // should be less than size for MI_BIN_HUGE
+#define MI_RETIRE_CYCLES      (16)
+
+// Retire a page with no more used blocks
+// Important to not retire too quickly though as new
+// allocations might coming.
+//
+// Note: called from `mi_free` and benchmarks often
+// trigger this due to freeing everything and then
+// allocating again so careful when changing this.
+void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {
+  mi_assert_internal(page != NULL);
+  mi_assert_expensive(_mi_page_is_valid(page));
+  mi_assert_internal(mi_page_all_free(page));
+
+  if (page->retire_expire!=0) return;  // already retired, just keep it retired
+  mi_page_set_has_interior_pointers(page, false);
+
+  // don't retire too often..
+  // (or we end up retiring and re-allocating most of the time)
+  // NOTE: refine this more: we should not retire if this
+  // is the only page left with free blocks. It is not clear
+  // how to check this efficiently though...
+  // for now, we don't retire if it is the only page left of this size class.
+  mi_page_queue_t* pq = mi_page_queue_of(page);
+  #if MI_RETIRE_CYCLES > 0
+  const size_t bsize = mi_page_block_size(page);
+  if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) {  // not full or huge queue?
+    if (pq->last==page && pq->first==page) { // the only page in the queue?
+      mi_theap_t* theap = mi_page_theap(page);
+      #if MI_STAT>0
+      mi_theap_stat_counter_increase(theap, pages_retire, 1);
+      #endif
+      page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4);
+      mi_assert_internal(pq >= theap->pages);
+      const size_t index = pq - theap->pages;
+      mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE);
+      if (index < theap->page_retired_min) theap->page_retired_min = index;
+      if (index > theap->page_retired_max) theap->page_retired_max = index;
+      mi_assert_internal(mi_page_all_free(page));
+      return; // don't free after all
+    }
+  }
+  #endif
+  _mi_page_free(page, pq);
+}
+
+// free retired pages: we don't need to look at the entire queues
+// since we only retire pages that are at the head position in a queue.
+void _mi_theap_collect_retired(mi_theap_t* theap, bool force) {
+  size_t min = MI_BIN_FULL;
+  size_t max = 0;
+  for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) {
+    mi_page_queue_t* pq   = &theap->pages[bin];
+    mi_page_t*       page = pq->first;
+    if (page != NULL && page->retire_expire != 0) {
+      if (mi_page_all_free(page)) {
+        page->retire_expire--;
+        if (page->retire_expire == 0 || force) {
+          _mi_page_free(page, pq);
+        }
+        else {
+          // keep retired, update min/max
+          if (bin < min) min = bin;
+          if (bin > max) max = bin;
+        }
+      }
+      else {
+        page->retire_expire = 0;
+      }
+    }
+  }
+  theap->page_retired_min = min;
+  theap->page_retired_max = max;
+}
+
+/*
+static void mi_theap_collect_full_pages(mi_theap_t* theap) {
+  // note: normally full pages get immediately abandoned and the full queue is always empty
+  // this path is only used if abandoning is disabled due to a destroy-able theap or options
+  // set by the user.
+  mi_page_queue_t* pq = &theap->pages[MI_BIN_FULL];
+  for (mi_page_t* page = pq->first; page != NULL; ) {
+    mi_page_t* next = page->next;         // get next in case we free the page
+    _mi_page_free_collect(page, false);   // register concurrent free's
+    // no longer full?
+    if (!mi_page_is_full(page)) {
+      if (mi_page_all_free(page)) {
+        _mi_page_free(page, pq);
+      }
+      else {
+        _mi_page_unfull(page);
+      }
+    }
+    page = next;
+  }
+}
+*/
+
+
+/* -----------------------------------------------------------
+  Initialize the initial free list in a page.
+  In secure mode we initialize a randomized list by
+  alternating between slices.
+----------------------------------------------------------- */
+
+#define MI_MAX_SLICE_SHIFT  (6)   // at most 64 slices
+#define MI_MAX_SLICES       (1UL << MI_MAX_SLICE_SHIFT)
+#define MI_MIN_SLICES       (2)
+
+static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* const page, const size_t bsize, const size_t extend) {
+  #if (MI_SECURE<3)
+  mi_assert_internal(page->free == NULL);
+  mi_assert_internal(page->local_free == NULL);
+  #endif
+  mi_assert_internal(page->capacity + extend <= page->reserved);
+  mi_assert_internal(bsize == mi_page_block_size(page));
+  void* const page_area = mi_page_start(page);
+
+  // initialize a randomized free list
+  // set up `slice_count` slices to alternate between
+  size_t shift = MI_MAX_SLICE_SHIFT;
+  while ((extend >> shift) == 0) {
+    shift--;
+  }
+  const size_t slice_count = (size_t)1U << shift;
+  const size_t slice_extend = extend / slice_count;
+  mi_assert_internal(slice_extend >= 1);
+  mi_block_t* blocks[MI_MAX_SLICES];   // current start of the slice
+  size_t      counts[MI_MAX_SLICES];   // available objects in the slice
+  for (size_t i = 0; i < slice_count; i++) {
+    blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i*slice_extend);
+    counts[i] = slice_extend;
+  }
+  counts[slice_count-1] += (extend % slice_count);  // final slice holds the modulus too (todo: distribute evenly?)
+
+  // and initialize the free list by randomly threading through them
+  // set up first element
+  const uintptr_t r = _mi_theap_random_next(theap);
+  size_t current = r % slice_count;
+  counts[current]--;
+  mi_block_t* const free_start = blocks[current];
+  // and iterate through the rest; use `random_shuffle` for performance
+  uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0
+  for (size_t i = 1; i < extend; i++) {
+    // call random_shuffle only every INTPTR_SIZE rounds
+    const size_t round = i%MI_INTPTR_SIZE;
+    if (round == 0) rnd = _mi_random_shuffle(rnd);
+    // select a random next slice index
+    size_t next = ((rnd >> 8*round) & (slice_count-1));
+    while (counts[next]==0) {                            // ensure it still has space
+      next++;
+      if (next==slice_count) next = 0;
+    }
+    // and link the current block to it
+    counts[next]--;
+    mi_block_t* const block = blocks[current];
+    blocks[current] = (mi_block_t*)((uint8_t*)block + bsize);  // bump to the following block
+    mi_block_set_next(page, block, blocks[next]);   // and set next; note: we may have `current == next`
+    current = next;
+  }
+  // prepend to the free list (usually NULL)
+  mi_block_set_next(page, blocks[current], page->free);  // end of the list
+  page->free = free_start;
+}
+
+static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend)
+{
+  #if (MI_SECURE<3)
+  mi_assert_internal(page->free == NULL);
+  mi_assert_internal(page->local_free == NULL);
+  #endif
+  mi_assert_internal(page->capacity + extend <= page->reserved);
+  mi_assert_internal(bsize == mi_page_block_size(page));
+  void* const page_area = mi_page_start(page);
+
+  mi_block_t* const start = mi_page_block_at(page, page_area, bsize, page->capacity);
+
+  // initialize a sequential free list
+  mi_block_t* const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1);
+  mi_block_t* block = start;
+  while(block <= last) {
+    mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize);
+    mi_block_set_next(page,block,next);
+    block = next;
+  }
+  // prepend to free list (usually `NULL`)
+  mi_block_set_next(page, last, page->free);
+  page->free = start;
+}
+
+/* -----------------------------------------------------------
+  Page initialize and extend the capacity
+----------------------------------------------------------- */
+
+#define MI_MAX_EXTEND_SIZE    (4*1024)      // heuristic, one OS page seems to work well.
+#if (MI_SECURE>=2)
+#define MI_MIN_EXTEND         (8*MI_SECURE) // extend at least by this many
+#else
+#define MI_MIN_EXTEND         (1)
+#endif
+
+// Extend the capacity (up to reserved) by initializing a free list
+// We do at most `MI_MAX_EXTEND` to avoid touching too much memory
+// Note: we also experimented with "bump" allocation on the first
+// allocations but this did not speed up any benchmark (due to an
+// extra test in malloc? or cache effects?)
+static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {
+  mi_assert_expensive(mi_page_is_valid_init(page));
+  #if (MI_SECURE<3)
+  mi_assert(page->free == NULL);
+  mi_assert(page->local_free == NULL);
+  if (page->free != NULL) return true;
+  #endif
+  if (page->capacity >= page->reserved) return true;
+
+  size_t page_size;
+  //uint8_t* page_start =
+  mi_page_area(page, &page_size);
+  #if MI_STAT>0
+  mi_theap_stat_counter_increase(theap, pages_extended, 1);
+  #endif
+
+  // calculate the extend count
+  const size_t bsize = mi_page_block_size(page);
+  size_t extend = (size_t)page->reserved - page->capacity;
+  mi_assert_internal(extend > 0);
+
+  size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/bsize);
+  if (max_extend < MI_MIN_EXTEND) { max_extend = MI_MIN_EXTEND; }
+  mi_assert_internal(max_extend > 0);
+
+  if (extend > max_extend) {
+    // ensure we don't touch memory beyond the page to reduce page commit.
+    // the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%.
+    extend = max_extend;
+  }
+
+  mi_assert_internal(extend > 0 && extend + page->capacity <= page->reserved);
+  mi_assert_internal(extend < (1UL<<16));
+
+  // commit on demand?
+  if (page->slice_committed > 0) {
+    const size_t needed_size = (page->capacity + extend)*bsize;
+    const size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), MI_PAGE_MIN_COMMIT_SIZE );
+    if (needed_commit > page->slice_committed) {
+      mi_assert_internal(((needed_commit - page->slice_committed) % _mi_os_page_size()) == 0);
+      if (!_mi_os_commit(mi_page_slice_start(page) + page->slice_committed, needed_commit - page->slice_committed, NULL)) {
+        return false;
+      }
+      page->slice_committed = needed_commit;
+    }
+  }
+
+  // and append the extend the free list
+  if (extend < MI_MIN_SLICES || MI_SECURE<2) { //!mi_option_is_enabled(mi_option_secure)) {
+    mi_page_free_list_extend(page, bsize, extend );
+  }
+  else {
+    mi_page_free_list_extend_secure(theap, page, bsize, extend);
+  }
+  // enable the new free list
+  page->capacity += (uint16_t)extend;
+  #if MI_STAT>0
+  mi_theap_stat_increase(theap, page_committed, extend * bsize);
+  #endif
+  mi_assert_expensive(mi_page_is_valid_init(page));
+  return true;
+}
+
+// Initialize a fresh page (that is already partially initialized)
+mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {
+  mi_assert(page != NULL);
+  mi_assert(theap!=NULL);
+  page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap`
+  mi_page_set_theap(page, theap);
+
+  size_t page_size;
+  uint8_t* page_start = mi_page_area(page, &page_size); MI_UNUSED(page_start);
+  mi_track_mem_noaccess(page_start,page_size);
+  mi_assert_internal(page_size / mi_page_block_size(page) < (1L<<16));
+  mi_assert_internal(page->reserved > 0);
+  #if (MI_PADDING || MI_ENCODE_FREELIST)
+  page->keys[0] = _mi_theap_random_next(theap);
+  page->keys[1] = _mi_theap_random_next(theap);
+  #endif
+  #if MI_DEBUG>2
+  if (page->memid.initially_zero) {
+    mi_track_mem_defined(page->page_start, mi_page_committed(page));
+    mi_assert_expensive(mi_mem_is_zero(page_start, mi_page_committed(page)));
+  }
+  #endif
+
+  mi_assert_internal(page->theap!=NULL);
+  mi_assert_internal(page->theap == mi_page_theap(page));
+  mi_assert_internal(page->capacity == 0);
+  mi_assert_internal(page->free == NULL);
+  mi_assert_internal(page->used == 0);
+  mi_assert_internal(mi_page_is_owned(page));
+  mi_assert_internal(page->xthread_free == 1);
+  mi_assert_internal(page->next == NULL);
+  mi_assert_internal(page->prev == NULL);
+  mi_assert_internal(page->retire_expire == 0);
+  mi_assert_internal(!mi_page_has_interior_pointers(page));
+  #if (MI_PADDING || MI_ENCODE_FREELIST)
+  mi_assert_internal(page->keys[0] != 0);
+  mi_assert_internal(page->keys[1] != 0);
+  #endif
+  mi_assert_expensive(mi_page_is_valid_init(page));
+
+  // initialize an initial free list
+  if (!mi_page_extend_free(theap,page)) return false;
+  mi_assert(mi_page_immediate_available(page));
+  return true;
+}
+
+
+/* -----------------------------------------------------------
+  Find pages with free blocks
+-------------------------------------------------------------*/
+
+// Find a page with free blocks of `page->block_size`.
+static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_page_queue_t* pq, bool first_try)
+{
+  // search through the pages in "next fit" order
+  size_t count = 0;
+  long candidate_limit = 0;          // we reset this on the first candidate to limit the search
+  long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages
+  mi_page_t* page_candidate = NULL;  // a page with free space
+  mi_page_t* page = pq->first;
+
+  while (page != NULL)
+  {
+    mi_page_t* next = page->next; // remember next (as this page can move to another queue)
+    count++;
+    candidate_limit--;
+
+    // search up to N pages for a best candidate
+
+    // is the local free list non-empty?
+    bool immediate_available = mi_page_immediate_available(page);
+    if (!immediate_available) {
+      // collect freed blocks by us and other threads to we get a proper use count
+      _mi_page_free_collect(page, false);
+      immediate_available = mi_page_immediate_available(page);
+    }
+
+    // if the page is completely full, move it to the `mi_pages_full`
+    // queue so we don't visit long-lived pages too often.
+    if (!immediate_available && !mi_page_is_expandable(page)) {
+      page_full_retain--;
+      if (page_full_retain < 0) {
+        mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));
+        mi_page_to_full(page, pq);
+      }
+    }
+    else {
+      // the page has free space, make it a candidate
+      // we prefer non-expandable pages with high usage as candidates (to reduce commit, and increase chances of free-ing up pages)
+      if (page_candidate == NULL) {
+        page_candidate = page;
+        candidate_limit = _mi_option_get_fast(mi_option_page_max_candidates);
+      }
+      else if (mi_page_all_free(page_candidate)) {
+        _mi_page_free(page_candidate, pq);
+        page_candidate = page;
+      }
+      // prefer to reuse fuller pages (in the hope the less used page gets freed)
+      else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) {
+        page_candidate = page;
+      }
+      // if we find a non-expandable candidate, or searched for N pages, return with the best candidate
+      if (immediate_available || candidate_limit <= 0) {
+        mi_assert_internal(page_candidate!=NULL);
+        break;
+      }
+    }
+
+  #if 0
+    // first-fit algorithm without candidates
+    // If the page contains free blocks, we are done
+    if (mi_page_immediate_available(page) || mi_page_is_expandable(page)) {
+      break;  // pick this one
+    }
+
+    // If the page is completely full, move it to the `mi_pages_full`
+    // queue so we don't visit long-lived pages too often.
+    mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));
+    mi_page_to_full(page, pq);
+  #endif
+
+    page = next;
+  } // for each page
+
+  mi_theap_stat_counter_increase(theap, page_searches, count);
+  mi_theap_stat_counter_increase(theap, page_searches_count, 1);
+
+  // set the page to the best candidate
+  if (page_candidate != NULL) {
+    page = page_candidate;
+  }
+  if (page != NULL) {
+    if (!mi_page_immediate_available(page)) {
+      mi_assert_internal(mi_page_is_expandable(page));
+      if (!mi_page_extend_free(theap, page)) {
+        page = NULL; // failed to extend
+      }
+    }
+    mi_assert_internal(page == NULL || mi_page_immediate_available(page));
+  }
+
+  if (page == NULL) {
+    _mi_theap_collect_retired(theap, false); // perhaps make a page available
+    page = mi_page_fresh(theap, pq);
+    mi_assert_internal(page == NULL || mi_page_immediate_available(page));
+    if (page == NULL && first_try) {
+      // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again
+      page = mi_page_queue_find_free_ex(theap, pq, false);
+      mi_assert_internal(page == NULL || mi_page_immediate_available(page));
+    }
+  }
+  else {
+    mi_assert_internal(page == NULL || mi_page_immediate_available(page));
+    // move the page to the front of the queue
+    mi_page_queue_move_to_front(theap, pq, page);
+    page->retire_expire = 0;
+    // _mi_theap_collect_retired(theap, false); // update retire counts; note: increases rss on MemoryLoad bench so don't do this
+  }
+  mi_assert_internal(page == NULL || mi_page_immediate_available(page));
+
+
+  return page;
+}
+
+
+
+// Find a page with free blocks of `size`.
+static mi_page_t* mi_find_free_page(mi_theap_t* theap, mi_page_queue_t* pq) {
+  // mi_page_queue_t* pq = mi_page_queue(theap, size);
+  mi_assert_internal(!mi_page_queue_is_huge(pq));
+
+  // check the first page: we even do this with candidate search or otherwise we re-search every time
+  mi_page_t* page = pq->first;
+  if mi_likely(page != NULL && mi_page_free_quick_collect(page)) {
+    #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness
+    if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) {
+      (void)mi_page_extend_free(theap, page);  // ok if this fails
+      mi_assert_internal(mi_page_immediate_available(page));
+    }
+    #endif
+    page->retire_expire = 0;
+    return page; // fast path
+  }
+  else {
+    return mi_page_queue_find_free_ex(theap, pq, true);
+  }
+}
+
+
+/* -----------------------------------------------------------
+  Users can register a deferred free function called
+  when the `free` list is empty. Since the `local_free`
+  is separate this is deterministically called after
+  a certain number of allocations.
+----------------------------------------------------------- */
+
+// The program should only install a single deferred free handler before doing allocation.
+static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers)
+static _Atomic(void*) deferred_arg;   
+
+void _mi_deferred_free(mi_theap_t* theap, bool force) {
+  theap->heartbeat++;
+  mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free);
+  if (fun != NULL && !theap->tld->recurse) {
+    theap->tld->recurse = true;
+    void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg);  
+    fun(force, theap->heartbeat, arg);
+    theap->tld->recurse = false;
+  }
+}
+
+void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {
+  mi_atomic_store_ptr_release(void,&deferred_arg, arg);
+  mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn);  
+}
+
+
+/* -----------------------------------------------------------
+  General allocation
+----------------------------------------------------------- */
+
+// Huge pages contain just one block, and the segment contains just that page.
+// Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX)
+// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`.
+static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page_alignment, mi_page_queue_t* pq) {
+  const size_t block_size = _mi_os_good_alloc_size(size);
+  // mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0);
+  #if MI_HUGE_PAGE_ABANDON
+  #error todo.
+  #else
+  // mi_page_queue_t* pq = mi_page_queue(theap, MI_LARGE_MAX_OBJ_SIZE+1);  // always in the huge queue regardless of the block size
+  mi_assert_internal(mi_page_queue_is_huge(pq));
+  #endif
+  mi_page_t* page = mi_page_fresh_alloc(theap, pq, block_size, page_alignment);
+  if (page != NULL) {
+    mi_assert_internal(mi_page_block_size(page) >= size);
+    mi_assert_internal(mi_page_immediate_available(page));
+    mi_assert_internal(mi_page_is_huge(page));
+    mi_assert_internal(mi_page_is_singleton(page));
+    #if MI_HUGE_PAGE_ABANDON
+    mi_assert_internal(mi_page_is_abandoned(page));
+    mi_page_set_theap(page, NULL);
+    #endif
+    mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page));
+    mi_theap_stat_counter_increase(theap, malloc_huge_count, 1);
+  }
+  return page;
+}
+
+
+// Allocate a page
+// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
+static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept {
+  const size_t req_size = size - MI_PADDING_SIZE;  // correct for padding_size in case of an overflow on `size`
+  if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) {
+    _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size);
+    return NULL;
+  }
+  mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size));
+  // huge allocation?
+  if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) {
+    return mi_huge_page_alloc(theap,size,huge_alignment,pq);
+  }
+  else {
+    // otherwise find a page with free blocks in our size segregated queues
+    #if MI_PADDING
+    mi_assert_internal(size >= MI_PADDING_SIZE);
+    #endif
+    return mi_find_free_page(theap, pq);
+  }
+}
+
+
+// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.
+// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
+// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for
+// very large requested alignments in which case we use a huge singleton page.
+// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit)
+// to use 4 parameters which compiles better on msvc for the malloc fast path.
+void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept
+{
+  const bool zero = ((zero_huge_alignment & 1) != 0);
+  const size_t huge_alignment = (zero_huge_alignment & ~1);
+
+  #if !MI_THEAP_INITASNULL
+  mi_assert_internal(theap != NULL);
+  #endif
+
+  // initialize if necessary
+  if mi_unlikely(!mi_theap_is_initialized(theap)) {
+    if (theap==&_mi_theap_empty_wrong) {
+      // we were unable to allocate a theap for a first-class heap
+      return NULL;
+    }
+    // otherwise we initialize the thread and its default theap
+    mi_thread_init();
+    theap = _mi_theap_default();
+    if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; }
+    mi_assert_internal(_mi_theap_default()==theap);
+  }
+  mi_assert_internal(mi_theap_is_initialized(theap));
+
+  // do administrative tasks every N generic mallocs
+  if mi_unlikely(++theap->generic_count >= 1000) {
+    theap->generic_collect_count += theap->generic_count;
+    theap->generic_count = 0;
+    // call potential deferred free routines
+    _mi_deferred_free(theap, false);
+    // free retired pages
+    _mi_theap_collect_retired(theap, false);
+
+    // collect every once in a while (10000 by default)
+    const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L);
+    if (theap->generic_collect_count >= generic_collect) {
+      theap->generic_collect_count = 0;
+      mi_theap_collect(theap, false /* force? */);
+    }
+  }
+
+  // find (or allocate) a page of the right size
+  mi_page_t* page = mi_find_page(theap, size, huge_alignment);
+  if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more
+    mi_theap_collect(theap, true /* force? */);
+    page = mi_find_page(theap, size, huge_alignment);
+  }
+
+  if mi_unlikely(page == NULL) { // out of memory
+    const size_t req_size = size - MI_PADDING_SIZE;  // correct for padding_size in case of an overflow on `size`
+    _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size);
+    return NULL;
+  }
+
+  mi_assert_internal(mi_page_immediate_available(page));
+  mi_assert_internal(mi_page_block_size(page) >= size);
+  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+
+  // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc)
+  if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
+  void* const p = _mi_page_malloc_zero(theap,page,size,zero);
+  mi_assert_internal(p != NULL);
+
+  // move full pages to the full queue
+  if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) {
+    mi_page_to_full(page, mi_page_queue_of(page));
+  }
+  return p;
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/emscripten/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/emscripten/prim.c
new file mode 100644
index 00000000..1ba0936e
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/emscripten/prim.c
@@ -0,0 +1,261 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen, Alon Zakai
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// This file is included in `src/prim/prim.c`
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/atomic.h"
+#include "mimalloc/prim.h"
+
+#include <sched.h>   // sched_yield
+#include <unistd.h>  // getentropy
+
+// Design
+// ======
+//
+// mimalloc is built on top of emmalloc. emmalloc is a minimal allocator on top
+// of sbrk. The reason for having three layers here is that we want mimalloc to
+// be able to allocate and release system memory properly, the same way it would
+// when using VirtualAlloc on Windows or mmap on POSIX, and sbrk is too limited.
+// Specifically, sbrk can only go up and down, and not "skip" over regions, and
+// so we end up either never freeing memory to the system, or we can get stuck
+// with holes.
+//
+// Atm wasm generally does *not* free memory back the system: once grown, we do
+// not shrink back down (https://github.com/WebAssembly/design/issues/1397).
+// However, that is expected to improve
+// (https://github.com/WebAssembly/memory-control/blob/main/proposals/memory-control/Overview.md)
+// and so we do not want to bake those limitations in here.
+//
+// Even without that issue, we want our system allocator to handle holes, that
+// is, it should merge freed regions and allow allocating new content there of
+// the full size, etc., so that we do not waste space. That means that the
+// system allocator really does need to handle the general problem of allocating
+// and freeing variable-sized chunks of memory in a random order, like malloc/
+// free do. And so it makes sense to layer mimalloc on top of such an
+// implementation.
+//
+// emmalloc makes sense for the lower level because it is small and simple while
+// still fully handling merging of holes etc. It is not the most efficient
+// allocator, but our assumption is that mimalloc needs to be fast while the
+// system allocator underneath it is called much less frequently.
+//
+
+//---------------------------------------------
+// init
+//---------------------------------------------
+
+void _mi_prim_mem_init( mi_os_mem_config_t* config) {
+  config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
+  config->alloc_granularity = 16;
+  config->has_overcommit = false;
+  config->has_partial_free = false;
+  config->has_virtual_reserve = false;
+}
+
+extern void emmalloc_free(void*);
+
+int _mi_prim_free(void* addr, size_t size) {
+  if (size==0) return 0;
+  emmalloc_free(addr);
+  return 0;
+}
+
+
+//---------------------------------------------
+// Allocation
+//---------------------------------------------
+
+extern void* emmalloc_memalign(size_t alignment, size_t size);
+
+// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
+int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
+  MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);
+  *is_large = false;
+  // TODO: Track the highest address ever seen; first uses of it are zeroes.
+  //       That assumes no one else uses sbrk but us (they could go up,
+  //       scribble, and then down), but we could assert on that perhaps.
+  *is_zero = false;
+  // emmalloc has a minimum alignment size.
+  #define MIN_EMMALLOC_ALIGN           8
+  if (try_alignment < MIN_EMMALLOC_ALIGN) {
+    try_alignment = MIN_EMMALLOC_ALIGN;
+  }
+  void* p = emmalloc_memalign(try_alignment, size);
+  *addr = p;
+  if (p == 0) {
+    return ENOMEM;
+  }
+  return 0;
+}
+
+
+//---------------------------------------------
+// Commit/Reset
+//---------------------------------------------
+
+int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  // See TODO above.
+  *is_zero = false;
+  return 0;
+}
+
+int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  *needs_recommit = false;
+  return 0;
+}
+
+int _mi_prim_reset(void* addr, size_t size) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  return 0;
+}
+
+int _mi_prim_reuse(void* addr, size_t size) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  return 0;
+}
+
+int _mi_prim_protect(void* addr, size_t size, bool protect) {
+  MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);
+  return 0;
+}
+
+
+//---------------------------------------------
+// Huge pages and NUMA nodes
+//---------------------------------------------
+
+int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
+  MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
+  *is_zero = true;
+  *addr = NULL;
+  return ENOSYS;
+}
+
+size_t _mi_prim_numa_node(void) {
+  return 0;
+}
+
+size_t _mi_prim_numa_node_count(void) {
+  return 1;
+}
+
+
+//----------------------------------------------------------------
+// Clock
+//----------------------------------------------------------------
+
+#include <emscripten/html5.h>
+
+mi_msecs_t _mi_prim_clock_now(void) {
+  return emscripten_date_now();
+}
+
+
+//----------------------------------------------------------------
+// Process info
+//----------------------------------------------------------------
+
+void _mi_prim_process_info(mi_process_info_t* pinfo)
+{
+  // use defaults
+  MI_UNUSED(pinfo);
+}
+
+
+//----------------------------------------------------------------
+// Output
+//----------------------------------------------------------------
+
+#include <emscripten/console.h>
+
+void _mi_prim_out_stderr( const char* msg) {
+  emscripten_console_error(msg);
+}
+
+
+//----------------------------------------------------------------
+// Environment
+//----------------------------------------------------------------
+
+bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+  // For code size reasons, do not support environ customization for now.
+  MI_UNUSED(name);
+  MI_UNUSED(result);
+  MI_UNUSED(result_size);
+  return false;
+}
+
+
+//----------------------------------------------------------------
+// Random
+//----------------------------------------------------------------
+
+bool _mi_prim_random_buf(void* buf, size_t buf_len) {
+  int err = getentropy(buf, buf_len);
+  return !err;
+}
+
+
+//----------------------------------------------------------------
+// Thread init/done
+//----------------------------------------------------------------
+
+#if defined(MI_USE_PTHREADS)
+
+// use pthread local storage keys to detect thread ending
+// (and used with MI_TLS_PTHREADS for the default theap)
+pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
+
+static void mi_pthread_done(void* value) {
+  if (value!=NULL) {
+    _mi_thread_done((mi_theap_t*)value);
+  }
+}
+
+void _mi_prim_thread_init_auto_done(void) {
+  mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));
+  pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);
+}
+
+void _mi_prim_thread_done_auto_done(void) {
+  if (_mi_heap_default_key != (pthread_key_t)(-1)) {  // do not leak the key, see issue #809
+    pthread_key_delete(_mi_heap_default_key);
+  }
+}
+
+void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+  if (_mi_heap_default_key != (pthread_key_t)(-1)) {  // can happen during recursive invocation on freeBSD
+    pthread_setspecific(_mi_heap_default_key, theap);
+  }
+}
+
+#else
+
+void _mi_prim_thread_init_auto_done(void) {
+  // nothing
+}
+
+void _mi_prim_thread_done_auto_done(void) {
+  // nothing
+}
+
+void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+  MI_UNUSED(theap);
+}
+#endif
+
+bool _mi_prim_thread_is_in_threadpool(void) {
+  return false;
+}
+
+void _mi_prim_thread_yield(void) {
+  sched_yield();
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/alloc-override-zone.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/alloc-override-zone.c
new file mode 100644
index 00000000..fdbe095c
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/alloc-override-zone.c
@@ -0,0 +1,481 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+
+#if defined(MI_MALLOC_OVERRIDE)
+
+#if !defined(__APPLE__)
+#error "this file should only be included on macOS"
+#endif
+
+/* ------------------------------------------------------
+   Override system malloc on macOS
+   This is done through the malloc zone interface.
+   It seems to be most robust in combination with interposing
+   though or otherwise we may get zone errors as there are could
+   be allocations done by the time we take over the
+   zone.
+------------------------------------------------------ */
+
+#include <AvailabilityMacros.h>
+#include <malloc/malloc.h>
+#include <string.h>  // memset
+#include <stdlib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
+// only available from OSX 10.6
+extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import));
+#endif
+
+/* ------------------------------------------------------
+   malloc zone members
+------------------------------------------------------ */
+
+static bool is_mimalloc_zone( malloc_zone_t* zone ); 
+
+static size_t zone_size(malloc_zone_t* zone, const void* p) {
+  if (mi_any_heap_contains(p)) { 
+    return mi_usable_size(p);
+  }
+  else if (!is_mimalloc_zone(zone)) {  // can happen due to interpose
+    return zone->size(zone,p);
+  }
+  else {
+    return 0;
+  }
+}
+
+static void* zone_malloc(malloc_zone_t* zone, size_t size) {
+  MI_UNUSED(zone);
+  return mi_malloc(size);
+}
+
+static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) {
+  MI_UNUSED(zone);
+  return mi_calloc(count, size);
+}
+
+static void* zone_valloc(malloc_zone_t* zone, size_t size) {
+  MI_UNUSED(zone);
+  return mi_malloc_aligned(size, _mi_os_page_size());
+}
+
+static void zone_free(malloc_zone_t* zone, void* p) {
+  if (mi_any_heap_contains(p)) {
+    mi_free(p); // with the page_map and pagemap_commit=1 we can use the regular free
+  }
+  else if (!is_mimalloc_zone(zone)) {  // can happen due to interpose
+    zone->free(zone,p);
+  }
+}
+
+static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) {
+  if (p == NULL || mi_any_heap_contains(p)) {
+    return mi_realloc(p, newsize);
+  }
+  else if (!is_mimalloc_zone(zone)) {  // can happen due to interpose
+    return zone->realloc(zone,p,newsize);
+  }
+  else {
+    return NULL;
+  }
+}
+
+static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) {
+  MI_UNUSED(zone);
+  return mi_malloc_aligned(size,alignment);
+}
+
+static void zone_destroy(malloc_zone_t* zone) {
+  if (!is_mimalloc_zone(zone)) {
+    zone->destroy(zone);
+  }
+}
+
+static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) {
+  unsigned i;
+  for (i = 0; i < count; i++) {
+    ps[i] = zone_malloc(zone, size);
+    if (ps[i] == NULL) break;
+  }
+  return i;
+}
+
+static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) {
+  for(size_t i = 0; i < count; i++) {
+    zone_free(zone, ps[i]);
+    ps[i] = NULL;
+  }
+}
+
+static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) {
+  MI_UNUSED(zone); MI_UNUSED(size);
+  mi_collect(false);
+  return 0;
+}
+
+static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) {
+  MI_UNUSED(size);
+  zone_free(zone,p);
+}
+
+static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) {
+  MI_UNUSED(zone);
+  return mi_is_in_heap_region(p);
+}
+
+
+/* ------------------------------------------------------
+   Introspection members
+------------------------------------------------------ */
+
+static kern_return_t intro_enumerator(task_t task, void* p,
+                            unsigned type_mask, vm_address_t zone_address,
+                            memory_reader_t reader,
+                            vm_range_recorder_t recorder)
+{
+  // todo: enumerate all memory
+  MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address);
+  MI_UNUSED(reader); MI_UNUSED(recorder);
+  return KERN_SUCCESS;
+}
+
+static size_t intro_good_size(malloc_zone_t* zone, size_t size) {
+  MI_UNUSED(zone);
+  return mi_good_size(size);
+}
+
+static boolean_t intro_check(malloc_zone_t* zone) {
+  MI_UNUSED(zone);
+  return true;
+}
+
+static void intro_print(malloc_zone_t* zone, boolean_t verbose) {
+  MI_UNUSED(zone); MI_UNUSED(verbose);
+  mi_stats_print(NULL);
+}
+
+static void intro_log(malloc_zone_t* zone, void* p) {
+  MI_UNUSED(zone); MI_UNUSED(p);
+  // todo?
+}
+
+static void intro_force_lock(malloc_zone_t* zone) {
+  MI_UNUSED(zone);
+  // todo?
+}
+
+static void intro_force_unlock(malloc_zone_t* zone) {
+  MI_UNUSED(zone);
+  // todo?
+}
+
+static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) {
+  MI_UNUSED(zone);
+  // todo...
+  stats->blocks_in_use = 0;
+  stats->size_in_use = 0;
+  stats->max_size_in_use = 0;
+  stats->size_allocated = 0;
+}
+
+static boolean_t intro_zone_locked(malloc_zone_t* zone) {
+  MI_UNUSED(zone);
+  return false;
+}
+
+
+/* ------------------------------------------------------
+  At process start, override the default allocator
+------------------------------------------------------ */
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+#endif
+
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wc99-extensions"
+#endif
+
+static malloc_introspection_t mi_introspect = {
+  .enumerator = &intro_enumerator,
+  .good_size = &intro_good_size,
+  .check = &intro_check,
+  .print = &intro_print,
+  .log = &intro_log,
+  .force_lock = &intro_force_lock,
+  .force_unlock = &intro_force_unlock,
+#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)
+  .statistics = &intro_statistics,
+  .zone_locked = &intro_zone_locked,
+#endif
+};
+
+static malloc_zone_t mi_malloc_zone = {
+  // note: even with designators, the order is important for C++ compilation
+  //.reserved1 = NULL,
+  //.reserved2 = NULL,
+  .size = &zone_size,
+  .malloc = &zone_malloc,
+  .calloc = &zone_calloc,
+  .valloc = &zone_valloc,
+  .free = &zone_free,
+  .realloc = &zone_realloc,
+  .destroy = &zone_destroy,
+  .zone_name = "mimalloc",
+  .batch_malloc = &zone_batch_malloc,
+  .batch_free = &zone_batch_free,
+  .introspect = &mi_introspect,
+#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)
+  #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)
+  .version = 10,
+  #else
+  .version = 9,
+  #endif
+  // switch to version 9+ on OSX 10.6 to support memalign.
+  .memalign = &zone_memalign,
+  .free_definite_size = &zone_free_definite_size,
+  #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
+  .pressure_relief = &zone_pressure_relief,
+  #endif
+  #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)
+  .claimed_address = &zone_claimed_address,
+  #endif
+#else
+  .version = 4,
+#endif
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+static bool is_mimalloc_zone( malloc_zone_t* zone ) {
+  return (zone==NULL || zone==&mi_malloc_zone);
+}
+
+#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT)
+
+// ------------------------------------------------------
+// Override malloc_xxx and malloc_zone_xxx api's to use only
+// our mimalloc zone. Since even the loader uses malloc
+// on macOS, this ensures that all allocations go through
+// mimalloc (as all calls are interposed).
+// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`,
+// Here, we also override macOS specific API's like
+// `malloc_zone_calloc` etc. see <https://github.com/aosm/libmalloc/blob/master/man/malloc_zone_malloc.3>
+// ------------------------------------------------------
+
+static inline malloc_zone_t* mi_get_default_zone(void) {
+  mi_atomic_do_once {
+    malloc_zone_register(&mi_malloc_zone);  // by calling register we avoid a zone error on free (see <http://eatmyrandom.blogspot.com/2010/03/mallocfree-interception-on-mac-os-x.html>)
+  }
+  return &mi_malloc_zone;
+}
+
+mi_decl_externc int  malloc_jumpstart(uintptr_t cookie);
+mi_decl_externc void _malloc_fork_prepare(void);
+mi_decl_externc void _malloc_fork_parent(void);
+mi_decl_externc void _malloc_fork_child(void);
+
+
+static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) {
+  MI_UNUSED(size); MI_UNUSED(flags);
+  return mi_get_default_zone();
+}
+
+static malloc_zone_t* mi_malloc_default_zone (void) {
+  return mi_get_default_zone();
+}
+
+static malloc_zone_t* mi_malloc_default_purgeable_zone(void) {
+  return mi_get_default_zone();
+}
+
+static void mi_malloc_destroy_zone(malloc_zone_t* zone) {
+  MI_UNUSED(zone);
+  // nothing.
+}
+
+static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) {
+  MI_UNUSED(task); MI_UNUSED(mr);
+  if (addresses != NULL) *addresses = NULL;
+  if (count != NULL) *count = 0;
+  return KERN_SUCCESS;
+}
+
+static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) {
+  return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name);
+}
+
+static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) {
+  MI_UNUSED(zone); MI_UNUSED(name);
+}
+
+static int mi_malloc_jumpstart(uintptr_t cookie) {
+  MI_UNUSED(cookie);
+  return 1; // or 0 for no error?
+}
+
+static void mi__malloc_fork_prepare(void) {
+  // nothing
+}
+static void mi__malloc_fork_parent(void) {
+  // nothing
+}
+static void mi__malloc_fork_child(void) {
+  // nothing
+}
+
+static void mi_malloc_printf(const char* fmt, ...) {
+  MI_UNUSED(fmt);
+}
+
+static bool zone_check(malloc_zone_t* zone) {
+  MI_UNUSED(zone);
+  return true;
+}
+
+static malloc_zone_t* zone_from_ptr(const void* p) {
+  MI_UNUSED(p);
+  return (mi_any_heap_contains(p) ? mi_get_default_zone() : NULL);
+}
+
+static void zone_log(malloc_zone_t* zone, void* p) {
+  MI_UNUSED(zone); MI_UNUSED(p);
+}
+
+static void zone_print(malloc_zone_t* zone, bool b) {
+  MI_UNUSED(zone); MI_UNUSED(b);
+}
+
+static void zone_print_ptr_info(void* p) {
+  MI_UNUSED(p);
+}
+
+static void zone_register(malloc_zone_t* zone) {
+  MI_UNUSED(zone);
+}
+
+static void zone_unregister(malloc_zone_t* zone) {
+  MI_UNUSED(zone);
+}
+
+// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`
+// See: <https://books.google.com/books?id=K8vUkpOXhN4C&pg=PA73>
+struct mi_interpose_s {
+  const void* replacement;
+  const void* target;
+};
+#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }
+#define MI_INTERPOSE_MI(fun)            MI_INTERPOSE_FUN(fun,mi_##fun)
+#define MI_INTERPOSE_ZONE(fun)          MI_INTERPOSE_FUN(malloc_##fun,fun)
+__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[]  __attribute__((section("__DATA, __interpose"))) =
+{
+
+  MI_INTERPOSE_MI(malloc_create_zone),
+  MI_INTERPOSE_MI(malloc_default_purgeable_zone),
+  MI_INTERPOSE_MI(malloc_default_zone),
+  MI_INTERPOSE_MI(malloc_destroy_zone),
+  MI_INTERPOSE_MI(malloc_get_all_zones),
+  MI_INTERPOSE_MI(malloc_get_zone_name),
+  MI_INTERPOSE_MI(malloc_jumpstart),
+  MI_INTERPOSE_MI(malloc_printf),
+  MI_INTERPOSE_MI(malloc_set_zone_name),
+  MI_INTERPOSE_MI(_malloc_fork_child),
+  MI_INTERPOSE_MI(_malloc_fork_parent),
+  MI_INTERPOSE_MI(_malloc_fork_prepare),
+
+  MI_INTERPOSE_ZONE(zone_batch_free),
+  MI_INTERPOSE_ZONE(zone_batch_malloc),
+  MI_INTERPOSE_ZONE(zone_calloc),
+  MI_INTERPOSE_ZONE(zone_check),
+  MI_INTERPOSE_ZONE(zone_free),
+  MI_INTERPOSE_ZONE(zone_from_ptr),
+  MI_INTERPOSE_ZONE(zone_log),
+  MI_INTERPOSE_ZONE(zone_malloc),
+  MI_INTERPOSE_ZONE(zone_memalign),
+  MI_INTERPOSE_ZONE(zone_print),
+  MI_INTERPOSE_ZONE(zone_print_ptr_info),
+  MI_INTERPOSE_ZONE(zone_realloc),
+  MI_INTERPOSE_ZONE(zone_register),
+  MI_INTERPOSE_ZONE(zone_unregister),
+  MI_INTERPOSE_ZONE(zone_valloc)
+};
+
+
+#else
+
+// ------------------------------------------------------
+// hook into the zone api's without interposing
+// This is the official way of adding an allocator but
+// it seems less robust than using interpose.
+// ------------------------------------------------------
+
+static inline malloc_zone_t* mi_get_default_zone(void)
+{
+  // The first returned zone is the real default
+  malloc_zone_t** zones = NULL;
+  unsigned count = 0;
+  kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count);
+  if (ret == KERN_SUCCESS && count > 0) {
+    return zones[0];
+  }
+  else {
+    // fallback
+    return malloc_default_zone();
+  }
+}
+
+#if defined(__clang__)
+__attribute__((constructor(101))) // highest priority
+#else
+__attribute__((constructor))      // priority level is not supported by gcc
+#endif
+__attribute__((used))
+static void _mi_macos_override_malloc(void) {
+  malloc_zone_t* purgeable_zone = NULL;
+
+  #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
+  // force the purgeable zone to exist to avoid strange bugs
+  if (malloc_default_purgeable_zone) {
+    purgeable_zone = malloc_default_purgeable_zone();
+  }
+  #endif
+
+  // Register our zone.
+  // thomcc: I think this is still needed to put us in the zone list.
+  malloc_zone_register(&mi_malloc_zone);
+  // Unregister the default zone, this makes our zone the new default
+  // as that was the last registered.
+  malloc_zone_t *default_zone = mi_get_default_zone();
+  // thomcc: Unsure if the next test is *always* false or just false in the
+  // cases I've tried. I'm also unsure if the code inside is needed. at all
+  if (default_zone != &mi_malloc_zone) {
+    malloc_zone_unregister(default_zone);
+
+    // Reregister the default zone so free and realloc in that zone keep working.
+    malloc_zone_register(default_zone);
+  }
+
+  // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs
+  // earlier than the default zone.
+  if (purgeable_zone != NULL) {
+    malloc_zone_unregister(purgeable_zone);
+    malloc_zone_register(purgeable_zone);
+  }
+
+}
+#endif  // MI_OSX_INTERPOSE
+
+#endif // MI_MALLOC_OVERRIDE
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/prim.c
new file mode 100644
index 00000000..8a2f4e8a
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/prim.c
@@ -0,0 +1,9 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// We use the unix/prim.c with the mmap API on macOSX
+#include "../unix/prim.c"
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/prim.c
new file mode 100644
index 00000000..5147bae8
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/prim.c
@@ -0,0 +1,76 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// Select the implementation of the primitives
+// depending on the OS.
+
+#if defined(_WIN32)
+#include "windows/prim.c"  // VirtualAlloc (Windows)
+
+#elif defined(__APPLE__)
+#include "osx/prim.c"      // macOSX (actually defers to mmap in unix/prim.c)
+
+#elif defined(__wasi__)
+#define MI_USE_SBRK
+#include "wasi/prim.c"     // memory-grow or sbrk (Wasm)
+
+#elif defined(__EMSCRIPTEN__)
+#include "emscripten/prim.c" // emmalloc_*, + pthread support
+
+#else
+#include "unix/prim.c"     // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.)
+
+#endif
+
+// Generic process initialization
+#ifndef MI_PRIM_HAS_PROCESS_ATTACH
+#if defined(__GNUC__) || defined(__clang__)
+  // gcc,clang: use the constructor/destructor attribute
+  // which for both seem to run before regular constructors/destructors
+  #if defined(__clang__)
+    #define mi_attr_constructor __attribute__((constructor(101)))
+    #define mi_attr_destructor  __attribute__((destructor(101)))
+  #else
+    #define mi_attr_constructor __attribute__((constructor))
+    #define mi_attr_destructor  __attribute__((destructor))
+  #endif
+  static void mi_attr_constructor mi_process_attach(void) {
+    _mi_auto_process_init();
+  }
+  static void mi_attr_destructor mi_process_detach(void) {
+    _mi_auto_process_done();
+  }
+#elif defined(__cplusplus)
+  // C++: use static initialization to detect process start/end
+  // This is not guaranteed to be first/last but the best we can generally do?
+  struct mi_init_done_t {
+    mi_init_done_t() {
+      _mi_auto_process_init();
+    }
+    ~mi_init_done_t() {
+      _mi_auto_process_done();
+    }
+  };
+  static mi_init_done_t mi_init_done;
+ #else
+  #pragma message("define a way to call _mi_auto_process_init/done on your platform")
+#endif
+#endif
+
+// Generic allocator init/done callback
+#ifndef MI_PRIM_HAS_ALLOCATOR_INIT
+bool _mi_is_redirected(void) {
+  return false;
+}
+bool _mi_allocator_init(const char** message) {
+  if (message != NULL) { *message = NULL; }
+  return true;
+}
+void _mi_allocator_done(void) {
+  // nothing to do
+}
+#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/readme.md b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/readme.md
new file mode 100644
index 00000000..380dd3a7
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/readme.md
@@ -0,0 +1,9 @@
+## Portability Primitives
+
+This is the portability layer where all primitives needed from the OS are defined.
+
+- `include/mimalloc/prim.h`: primitive portability API definition.
+- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform
+            (and on macOS, `osx/prim.c` defers to `unix/prim.c`).
+
+Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's.
\ No newline at end of file
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c
new file mode 100644
index 00000000..43efea36
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c
@@ -0,0 +1,1001 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// This file is included in `src/prim/prim.c`
+
+#ifndef _DEFAULT_SOURCE
+#define _DEFAULT_SOURCE   // ensure mmap flags and syscall are defined
+#endif
+
+#if defined(__sun)
+// illumos provides new mman.h api when any of these are defined
+// otherwise the old api based on caddr_t which predates the void pointers one.
+// stock solaris provides only the former, chose to atomically to discard those
+// flags only here rather than project wide tough.
+#undef _XOPEN_SOURCE
+#undef _POSIX_C_SOURCE
+#endif
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"
+
+#include <sys/mman.h>  // mmap
+#include <unistd.h>    // sysconf, sleep
+#include <fcntl.h>     // open, close, read, access
+#include <stdlib.h>    // getenv, arc4random_buf
+
+#if defined(__linux__)
+  #include <features.h>
+  #include <sys/prctl.h>    // THP disable, PR_SET_VMA
+  #include <sys/sysinfo.h>  // sysinfo
+  #if defined(__GLIBC__) && !defined(PR_SET_VMA)
+  #include <linux/prctl.h>
+  #endif
+  #if defined(__GLIBC__)
+  #include <linux/mman.h>   // linux mmap flags
+  #else
+  #include <sys/mman.h>
+  #endif
+#elif defined(__APPLE__)
+  #include <AvailabilityMacros.h>
+  #include <TargetConditionals.h>
+  #if !defined(TARGET_OS_OSX) || TARGET_OS_OSX   // see issue #879, used to be (!TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR)
+  #include <mach/vm_statistics.h>    // VM_MAKE_TAG, VM_FLAGS_SUPERPAGE_SIZE_2MB, etc.
+  #endif
+  #if !defined(MAC_OS_X_VERSION_10_7)
+  #define MAC_OS_X_VERSION_10_7   1070
+  #endif
+  #include <sys/sysctl.h>
+#elif defined(__FreeBSD__) || defined(__DragonFly__)
+  #include <sys/param.h>
+  #if __FreeBSD_version >= 1200000
+  #include <sys/cpuset.h>
+  #include <sys/domainset.h>
+  #endif
+  #include <sys/sysctl.h>
+#endif
+
+#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__FreeBSD__)
+  #define MI_HAS_SYSCALL_H
+  #include <sys/syscall.h>
+#endif
+
+#if !defined(MADV_DONTNEED) && defined(POSIX_MADV_DONTNEED)  // QNX
+#define MADV_DONTNEED  POSIX_MADV_DONTNEED
+#endif
+#if !defined(MADV_FREE) && defined(POSIX_MADV_FREE)  // QNX
+#define MADV_FREE  POSIX_MADV_FREE
+#endif
+
+#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // TODO: can we query the OS for this?
+
+//------------------------------------------------------------------------------------
+// Use syscalls for some primitives to allow for libraries that override open/read/close etc.
+// and do allocation themselves; using syscalls prevents recursion when mimalloc is
+// still initializing (issue #713)
+// Declare inline to avoid unused function warnings.
+//------------------------------------------------------------------------------------
+
+#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access)
+
+static inline int mi_prim_open(const char* fpath, int open_flags) {
+  return syscall(SYS_open,fpath,open_flags,0);
+}
+static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
+  return syscall(SYS_read,fd,buf,bufsize);
+}
+static inline int mi_prim_close(int fd) {
+  return syscall(SYS_close,fd);
+}
+static inline int mi_prim_access(const char *fpath, int mode) {
+  return syscall(SYS_access,fpath,mode);
+}
+
+#else
+
+static inline int mi_prim_open(const char* fpath, int open_flags) {
+  return open(fpath,open_flags);
+}
+static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
+  return read(fd,buf,bufsize);
+}
+static inline int mi_prim_close(int fd) {
+  return close(fd);
+}
+static inline int mi_prim_access(const char *fpath, int mode) {
+  return access(fpath,mode);
+}
+
+#endif
+
+
+
+//---------------------------------------------
+// init
+//---------------------------------------------
+
+static bool unix_detect_overcommit(void) {
+  bool os_overcommit = true;
+  #if defined(__linux__)
+    int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY);
+    if (fd >= 0) {
+      char buf[32];
+      ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));
+      mi_prim_close(fd);
+      // <https://www.kernel.org/doc/Documentation/vm/overcommit-accounting>
+      // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE)
+      if (nread >= 1) {
+        os_overcommit = (buf[0] == '0' || buf[0] == '1');
+      }
+    }
+  #elif defined(__FreeBSD__)
+    int val = 0;
+    size_t olen = sizeof(val);
+    if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) {
+      os_overcommit = (val != 0);
+    }
+  #else
+    // default: overcommit is true
+  #endif
+  return os_overcommit;
+}
+
+static bool unix_detect_thp(void) {
+  bool thp_enabled = false;
+  #if defined(__linux__)
+  int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
+  if (fd >= 0) {
+    char buf[32];
+    ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));
+    mi_prim_close(fd);
+    // <https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html>
+    // between brackets is the current value, for example: always [madvise] never
+    if (nread >= 1) {
+      thp_enabled = (_mi_strnstr(buf,32,"[never]") == NULL);
+    }
+  }
+  #endif
+  return thp_enabled;
+}
+
+// try to detect the physical memory dynamically (if possible)
+static void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) {
+  #if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE))  // freeBSD, macOS
+    MI_UNUSED(page_size);
+    int64_t physical_memory = 0;
+    size_t length = sizeof(int64_t);
+    #if defined(HW_PHYSMEM64)
+    int mib[2] = { CTL_HW, HW_PHYSMEM64 };
+    #else
+    int mib[2] = { CTL_HW, HW_MEMSIZE };
+    #endif
+    const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0);
+    if (err==0 && physical_memory > 0) {
+      const int64_t phys_in_kib = physical_memory / MI_KiB;
+      if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) {
+        *physical_memory_in_kib = (size_t)phys_in_kib;
+      }
+    }
+  #elif defined(__linux__)
+    MI_UNUSED(page_size);
+    struct sysinfo info; _mi_memzero_var(info);
+    const int err = sysinfo(&info);
+    if (err==0 && info.mem_unit > 0 && info.totalram <= SIZE_MAX) {
+      if (info.mem_unit==MI_KiB) {
+        *physical_memory_in_kib = (size_t)info.totalram;
+      }
+      else {
+        size_t total = 0;
+        if (!mi_mul_overflow((size_t)info.totalram, (size_t)info.mem_unit, &total)) {
+          *physical_memory_in_kib = (total / MI_KiB);
+        }
+      }
+    }
+  #elif defined(_SC_PHYS_PAGES)  // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100)
+    const long pphys = sysconf(_SC_PHYS_PAGES);
+    const size_t psize_in_kib = page_size / MI_KiB;
+    if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) {
+      *physical_memory_in_kib = (size_t)pphys * psize_in_kib;
+    }
+  #endif
+}
+
+void _mi_prim_mem_init( mi_os_mem_config_t* config )
+{
+  long psize = sysconf(_SC_PAGESIZE);
+  if (psize > 0 && (unsigned long)psize < SIZE_MAX) {
+    config->page_size = (size_t)psize;
+    config->alloc_granularity = (size_t)psize;
+    unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib);
+  }
+  config->large_page_size = MI_UNIX_LARGE_PAGE_SIZE;
+  config->has_overcommit = unix_detect_overcommit();
+  config->has_partial_free = true;    // mmap can free in parts
+  config->has_virtual_reserve = true; // todo: check if this true for NetBSD?  (for anonymous mmap with PROT_NONE)
+  config->has_transparent_huge_pages = unix_detect_thp();
+
+  // disable transparent huge pages for this process?
+  #if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE)
+  if (!mi_option_is_enabled(mi_option_allow_thp)) // disable THP if requested through an option
+  {
+    config->has_transparent_huge_pages = false;
+    if (prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0) == 0) {   // -1 on error, 1 if already disabled
+      // Most likely since distros often come with always/madvise settings.
+      // Disabling only for mimalloc process rather than touching system wide settings
+      (void)prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0);
+    }
+  }
+  #endif
+}
+
+
+//---------------------------------------------
+// free
+//---------------------------------------------
+
+int _mi_prim_free(void* addr, size_t size ) {
+  if (size==0) return 0;
+  bool err = (munmap(addr, size) == -1);
+  return (err ? errno : 0);
+}
+
+
+//---------------------------------------------
+// mmap
+//---------------------------------------------
+
+// return errno on failure
+static int unix_madvise(void* addr, size_t size, int advice) {
+  #if defined(__sun)
+  const int res = madvise((caddr_t)addr, size, advice);  // Solaris needs cast (issue #520)
+  return (res==0 ? 0 : errno);
+  #elif defined(__QNX__)
+  return posix_madvise(addr, size, advice);              // posix returns errno
+  #else
+  const int res = madvise(addr, size, advice);           // linux returns -1 on failure and sets errno
+  return (res==0 ? 0 : errno);
+  #endif
+}
+
+static void* unix_mmap_prim(void* addr, size_t size, int protect_flags, int flags, int fd) {
+  void* p = mmap(addr, size, protect_flags, flags, fd, 0 /* offset */);
+  #if defined(__linux__) && defined(PR_SET_VMA)
+  if (p!=MAP_FAILED && p!=NULL) {
+    prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, size, "mimalloc");
+  }
+  #endif
+  return p;
+}
+
+static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
+  MI_UNUSED(try_alignment);
+  void* p = NULL;
+  #if defined(MAP_ALIGNED)  // BSD
+  if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
+    size_t n = 0;
+    mi_bsr(try_alignment, &n);
+    if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) {  // alignment is a power of 2 and 4096 <= alignment <= 1GiB
+      p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd);
+      if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
+        int err = errno;
+        _mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
+      }
+      if (p!=MAP_FAILED) return p;
+      // fall back to regular mmap
+    }
+  }
+  #elif defined(MAP_ALIGN)  // Solaris
+  if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
+    p = unix_mmap_prim((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd);  // addr parameter is the required alignment
+    if (p!=MAP_FAILED) return p;
+    // fall back to regular mmap
+  }
+  #endif
+  #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
+  // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations
+  if (addr == NULL) {
+    void* hint = _mi_os_get_aligned_hint(try_alignment, size);
+    if (hint != NULL) {
+      p = unix_mmap_prim(hint, size, protect_flags, flags, fd);
+      if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
+        #if MI_TRACK_ENABLED  // asan sometimes does not instrument errno correctly?
+        int err = 0;
+        #else
+        int err = errno;
+        #endif
+        _mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
+      }
+      if (p!=MAP_FAILED) return p;
+      // fall back to regular mmap
+    }
+  }
+  #endif
+  // regular mmap
+  p = unix_mmap_prim(addr, size, protect_flags, flags, fd);
+  if (p!=MAP_FAILED) return p;
+  // failed to allocate
+  return NULL;
+}
+
+static int unix_mmap_fd(void) {
+  #if defined(VM_MAKE_TAG)
+  // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
+  int os_tag = (int)mi_option_get(mi_option_os_tag);
+  if (os_tag < 100 || os_tag > 255) { os_tag = 254; }
+  return VM_MAKE_TAG(os_tag);
+  #else
+  return -1;
+  #endif
+}
+
+static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
+  #if !defined(MAP_ANONYMOUS)
+  #define MAP_ANONYMOUS  MAP_ANON
+  #endif
+  #if !defined(MAP_NORESERVE)
+  #define MAP_NORESERVE  0
+  #endif
+  void* p = NULL;
+  const int fd = unix_mmap_fd();
+  int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+  if (_mi_os_has_overcommit()) {
+    flags |= MAP_NORESERVE;
+  }
+  #if defined(PROT_MAX)
+  protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
+  #endif
+  // huge page allocation
+  if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) {
+    static _Atomic(size_t) large_page_try_ok; // = 0;
+    size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
+    if (!large_only && try_ok > 0) {
+      // If the OS is not configured for large OS pages, or the user does not have
+      // enough permission, the `mmap` will always fail (but it might also fail for other reasons).
+      // Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times
+      // to avoid too many failing calls to mmap.
+      mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);
+    }
+    else {
+      int lflags = flags & ~MAP_NORESERVE;  // using NORESERVE on huge pages seems to fail on Linux
+      int lfd = fd;
+      #ifdef MAP_ALIGNED_SUPER
+      lflags |= MAP_ALIGNED_SUPER;
+      #endif
+      #ifdef MAP_HUGETLB
+      lflags |= MAP_HUGETLB;
+      #endif
+      #ifdef MAP_HUGE_1GB
+      static bool mi_huge_pages_available = true;
+      if (large_only && (size % MI_GiB) == 0 && mi_huge_pages_available) {
+        lflags |= MAP_HUGE_1GB;
+      }
+      else
+      #endif
+      {
+        #ifdef MAP_HUGE_2MB
+        lflags |= MAP_HUGE_2MB;
+        #endif
+      }
+      #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB
+      lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
+      #endif
+      if (large_only || lflags != flags) {
+        // try large OS page allocation
+        *is_large = true;
+        p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
+        #ifdef MAP_HUGE_1GB
+        if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {
+          mi_huge_pages_available = false; // don't try huge 1GiB pages again
+          if (large_only) {
+            _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno);
+          }
+          lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
+          p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
+        }
+        #endif
+        if (large_only) return p;
+        if (p == NULL) {
+          mi_atomic_store_release(&large_page_try_ok, (size_t)8);  // on error, don't try again for the next N allocations
+        }
+      }
+    }
+  }
+  // regular allocation
+  if (p == NULL) {
+    *is_large = false;
+    p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd);
+    #if !defined(MI_NO_THP)
+    if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) {
+      #if defined(MADV_HUGEPAGE)
+      // Many Linux systems don't allow MAP_HUGETLB but they support instead
+      // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE
+      // though since properly aligned allocations will already use large pages if available
+      // in that case -- in particular for our large regions (in `memory.c`).
+      // However, some systems only allow THP if called with explicit `madvise`, so
+      // when large OS pages are enabled for mimalloc, we call `madvise` anyways.
+      if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {
+        // *is_large = true; // possibly
+      };
+      #elif defined(__sun)
+      struct memcntl_mha cmd = {0};
+      cmd.mha_pagesize = _mi_os_large_page_size();
+      cmd.mha_cmd = MHA_MAPSIZE_VA;
+      if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
+        // *is_large = true; // possibly
+      }
+      #endif
+    }
+    #endif
+  }
+  return p;
+}
+
+// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
+int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
+  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
+  mi_assert_internal(commit || !allow_large);
+  mi_assert_internal(try_alignment > 0);
+  *is_zero = true;
+  int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
+  *addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large);
+  return (*addr != NULL ? 0 : errno);
+}
+
+
+//---------------------------------------------
+// Commit/Reset
+//---------------------------------------------
+
+static void unix_mprotect_hint(int err) {
+  #if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page
+  if (err == ENOMEM) {
+    _mi_warning_message("The next warning may be caused by a low memory map limit.\n"
+                        "  On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n"
+                        "  For example: sudo sysctl -w vm.max_map_count=262144\n");
+  }
+  #else
+  MI_UNUSED(err);
+  #endif
+}
+
+
+
+
+
+int _mi_prim_commit(void* start, size_t size, bool* is_zero) {
+  // commit: ensure we can access the area
+  // note: we may think that *is_zero can be true since the memory
+  // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but
+  // we sometimes call commit on a range with still partially committed
+  // memory and `mprotect` does not zero the range.
+  *is_zero = false;
+  int err = mprotect(start, size, (PROT_READ | PROT_WRITE));
+  if (err != 0) {
+    err = errno;
+    unix_mprotect_hint(err);
+  }
+  return err;
+}
+
+int _mi_prim_reuse(void* start, size_t size) {
+  MI_UNUSED(start); MI_UNUSED(size);
+  #if defined(__APPLE__) && defined(MADV_FREE_REUSE)
+  return unix_madvise(start, size, MADV_FREE_REUSE);
+  #endif
+  return 0;
+}
+
+int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
+  int err = 0;
+  #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
+    // decommit on macOS: map fresh zero pages over the range (WeavePy patch).
+    // MADV_FREE_REUSABLE only lowers the footprint: the pages stay in the
+    // resident size (and `ru_maxrss`) until the system reclaims them. If the
+    // mapping fails, the range is left as it was and marked reusable.
+    void* p = mmap(start, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, unix_mmap_fd(), 0);
+    if (p != start) {
+      err = unix_madvise(start, size, MADV_FREE_REUSABLE);
+      if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
+    }
+  #else
+    // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
+    err = unix_madvise(start, size, MADV_DONTNEED);
+  #endif
+  #if !MI_DEBUG && MI_SECURE<=2
+    *needs_recommit = false;
+  #else
+    *needs_recommit = true;
+    mprotect(start, size, PROT_NONE);
+  #endif
+  /*
+  // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)
+  *needs_recommit = true;
+  const int fd = unix_mmap_fd();
+  void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
+  if (p != start) { err = errno; }
+  */
+  return err;
+}
+
+int _mi_prim_reset(void* start, size_t size) {
+  int err = 0;
+
+  // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower)
+  #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
+  err = unix_madvise(start, size, MADV_FREE_REUSABLE);
+  if (err==0) return 0;
+  // fall through
+  #endif
+
+  #if defined(MADV_FREE)
+  // Otherwise, we try to use `MADV_FREE` as that is the fastest. A drawback though is that it
+  // will not reduce the `rss` stats in tools like `top` even though the memory is available
+  // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by
+  // default `MADV_DONTNEED` is used though.
+  static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE);
+  int oadvice = (int)mi_atomic_load_relaxed(&advice);
+  while ((err = unix_madvise(start, size, oadvice)) != 0 && err == EAGAIN) { /* try again */ };
+  if (err == EINVAL && oadvice == MADV_FREE) {
+    // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
+    mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED);
+    err = unix_madvise(start, size, MADV_DONTNEED);
+  }
+  #else
+  err = unix_madvise(start, size, MADV_DONTNEED);
+  #endif
+  return err;
+}
+
+int _mi_prim_protect(void* start, size_t size, bool protect) {
+  int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));
+  if (err != 0) { err = errno; }
+  unix_mprotect_hint(err);
+  return err;
+}
+
+
+
+//---------------------------------------------
+// Huge page allocation
+//---------------------------------------------
+
+#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)
+
+#ifndef MPOL_PREFERRED
+#define MPOL_PREFERRED 1
+#endif
+
+#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind)
+static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
+  return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags);
+}
+#else
+static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
+  MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags);
+  return 0;
+}
+#endif
+
+int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
+  bool is_large = true;
+  *is_zero = true;
+  *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large);
+  if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes
+    unsigned long numa_mask = (1UL << numa_node);
+    // TODO: does `mbind` work correctly for huge OS pages? should we
+    // use `set_mempolicy` before calling mmap instead?
+    // see: <https://lkml.org/lkml/2017/2/9/875>
+    long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);
+    if (err != 0) {
+      err = errno;
+      _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err);
+    }
+  }
+  return (*addr != NULL ? 0 : errno);
+}
+
+#else
+
+int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
+  MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
+  *is_zero = false;
+  *addr = NULL;
+  return ENOMEM;
+}
+
+#endif
+
+//---------------------------------------------
+// NUMA nodes
+//---------------------------------------------
+
+#if defined(__linux__)
+
+size_t _mi_prim_numa_node(void) {
+  #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)
+    unsigned long node = 0;
+    unsigned long ncpu = 0;
+    long err = syscall(SYS_getcpu, &ncpu, &node, NULL);
+    if (err != 0) return 0;
+    return node;
+  #else
+    return 0;
+  #endif
+}
+
+size_t _mi_prim_numa_node_count(void) {
+  char buf[128];
+  unsigned node = 0;
+  for(node = 0; node < 256; node++) {
+    // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
+    _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
+    if (mi_prim_access(buf,R_OK) != 0) break;
+  }
+  return (node+1);
+}
+
+#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000
+
+size_t _mi_prim_numa_node(void) {
+  domainset_t dom;
+  size_t node;
+  int policy;
+  if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul;
+  for (node = 0; node < MAXMEMDOM; node++) {
+    if (DOMAINSET_ISSET(node, &dom)) return node;
+  }
+  return 0ul;
+}
+
+size_t _mi_prim_numa_node_count(void) {
+  size_t ndomains = 0;
+  size_t len = sizeof(ndomains);
+  if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul;
+  return ndomains;
+}
+
+#elif defined(__DragonFly__)
+
+size_t _mi_prim_numa_node(void) {
+  // TODO: DragonFly does not seem to provide any userland means to get this information.
+  return 0ul;
+}
+
+size_t _mi_prim_numa_node_count(void) {
+  size_t ncpus = 0, nvirtcoresperphys = 0;
+  size_t len = sizeof(size_t);
+  if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul;
+  if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul;
+  return nvirtcoresperphys * ncpus;
+}
+
+#else
+
+size_t _mi_prim_numa_node(void) {
+  return 0;
+}
+
+size_t _mi_prim_numa_node_count(void) {
+  return 1;
+}
+
+#endif
+
+// ----------------------------------------------------------------
+// Clock
+// ----------------------------------------------------------------
+
+#include <time.h>
+
+#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
+
+mi_msecs_t _mi_prim_clock_now(void) {
+  struct timespec t;
+  #ifdef CLOCK_MONOTONIC
+  clock_gettime(CLOCK_MONOTONIC, &t);
+  #else
+  clock_gettime(CLOCK_REALTIME, &t);
+  #endif
+  return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);
+}
+
+#else
+
+// low resolution timer
+mi_msecs_t _mi_prim_clock_now(void) {
+  #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)
+  return (mi_msecs_t)clock();
+  #elif (CLOCKS_PER_SEC < 1000)
+  return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
+  #else
+  return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
+  #endif
+}
+
+#endif
+
+
+
+
+//----------------------------------------------------------------
+// Process info
+//----------------------------------------------------------------
+
+#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/resource.h>
+
+#if defined(__APPLE__)
+#include <mach/mach.h>
+#endif
+
+#if defined(__HAIKU__)
+#include <kernel/OS.h>
+#endif
+
+static mi_msecs_t timeval_secs(const struct timeval* tv) {
+  return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L);
+}
+
+void _mi_prim_process_info(mi_process_info_t* pinfo)
+{
+  struct rusage rusage;
+  getrusage(RUSAGE_SELF, &rusage);
+  pinfo->utime = timeval_secs(&rusage.ru_utime);
+  pinfo->stime = timeval_secs(&rusage.ru_stime);
+#if !defined(__HAIKU__)
+  pinfo->page_faults = rusage.ru_majflt;
+#endif
+#if defined(__HAIKU__)
+  // Haiku does not have (yet?) a way to
+  // get these stats per process
+  thread_info tid;
+  area_info mem;
+  ssize_t c;
+  get_thread_info(find_thread(0), &tid);
+  while (get_next_area_info(tid.team, &c, &mem) == B_OK) {
+    pinfo->peak_rss += mem.ram_size;
+  }
+  pinfo->page_faults = 0;
+#elif defined(__APPLE__)
+  pinfo->peak_rss = rusage.ru_maxrss;         // macos reports in bytes
+  #ifdef MACH_TASK_BASIC_INFO
+  struct mach_task_basic_info info;
+  mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
+  if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
+    pinfo->current_rss = (size_t)info.resident_size;
+  }
+  #else
+  struct task_basic_info info;
+  mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;
+  if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
+    pinfo->current_rss = (size_t)info.resident_size;
+  }
+  #endif
+#else
+  pinfo->peak_rss = rusage.ru_maxrss * 1024;  // Linux/BSD report in KiB
+#endif
+  // use defaults for commit
+}
+
+#else
+
+#ifndef __wasi__
+// WebAssembly instances are not processes
+#pragma message("define a way to get process info")
+#endif
+
+void _mi_prim_process_info(mi_process_info_t* pinfo)
+{
+  // use defaults
+  MI_UNUSED(pinfo);
+}
+
+#endif
+
+
+//----------------------------------------------------------------
+// Output
+//----------------------------------------------------------------
+
+void _mi_prim_out_stderr( const char* msg ) {
+  fputs(msg,stderr);
+}
+
+
+//----------------------------------------------------------------
+// Environment
+//----------------------------------------------------------------
+
+#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0)
+// On Posix systemsr use `environ` to access environment variables
+// even before the C runtime is initialized.
+#if defined(__APPLE__) && defined(__has_include) && __has_include(<crt_externs.h>)
+#include <crt_externs.h>
+static char** mi_get_environ(void) {
+  return (*_NSGetEnviron());
+}
+#else
+extern char** environ;
+static char** mi_get_environ(void) {
+  return environ;
+}
+#endif
+bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+  if (name==NULL) return false;
+  const size_t len = _mi_strlen(name);
+  if (len == 0) return false;
+  char** env = mi_get_environ();
+  if (env == NULL) return false;
+  // compare up to 10000 entries
+  for (int i = 0; i < 10000 && env[i] != NULL; i++) {
+    const char* s = env[i];
+    if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive
+      // found it
+      _mi_strlcpy(result, s + len + 1, result_size);
+      return true;
+    }
+  }
+  return false;
+}
+#else
+// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime
+bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+  // cannot call getenv() when still initializing the C runtime.
+  if (_mi_preloading()) return false;
+  const char* s = getenv(name);
+  if (s == NULL) {
+    // we check the upper case name too.
+    char buf[64+1];
+    size_t len = _mi_strnlen(name,sizeof(buf)-1);
+    for (size_t i = 0; i < len; i++) {
+      buf[i] = _mi_toupper(name[i]);
+    }
+    buf[len] = 0;
+    s = getenv(buf);
+  }
+  if (s == NULL || _mi_strnlen(s,result_size) >= result_size)  return false;
+  _mi_strlcpy(result, s, result_size);
+  return true;
+}
+#endif  // !MI_USE_ENVIRON
+
+
+//----------------------------------------------------------------
+// Random
+//----------------------------------------------------------------
+
+#if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15)
+#include <CommonCrypto/CommonCryptoError.h>
+#include <CommonCrypto/CommonRandom.h>
+
+bool _mi_prim_random_buf(void* buf, size_t buf_len) {
+  // We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf
+  // may fail silently on macOS. See PR #390, and <https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html>
+  return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess);
+}
+
+#elif defined(__ANDROID__) || defined(__DragonFly__) || \
+      defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+      defined(__sun) || \
+      (defined(__APPLE__) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_7))
+
+bool _mi_prim_random_buf(void* buf, size_t buf_len) {
+  arc4random_buf(buf, buf_len);
+  return true;
+}
+
+#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__)   // also for old apple versions < 10.7 (issue #829)
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+
+bool _mi_prim_random_buf(void* buf, size_t buf_len) {
+  // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h`
+  // and for the latter the actual `getrandom` call is not always defined.
+  // (see <https://stackoverflow.com/questions/45237324/why-doesnt-getrandom-compile>)
+  // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed.
+  #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom)
+    #ifndef GRND_NONBLOCK
+    #define GRND_NONBLOCK (1)
+    #endif
+    static _Atomic(uintptr_t) no_getrandom; // = 0
+    if (mi_atomic_load_acquire(&no_getrandom)==0) {
+      ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK);
+      if (ret >= 0) return (buf_len == (size_t)ret);
+      if (errno != ENOSYS) return false;
+      mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom
+    }
+  #endif
+  int flags = O_RDONLY;
+  #if defined(O_CLOEXEC)
+  flags |= O_CLOEXEC;
+  #endif
+  int fd = mi_prim_open("/dev/urandom", flags);
+  if (fd < 0) return false;
+  size_t count = 0;
+  while(count < buf_len) {
+    ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count);
+    if (ret<=0) {
+      if (errno!=EAGAIN && errno!=EINTR) break;
+    }
+    else {
+      count += ret;
+    }
+  }
+  mi_prim_close(fd);
+  return (count==buf_len);
+}
+
+#else
+
+bool _mi_prim_random_buf(void* buf, size_t buf_len) {
+  return false;
+}
+
+#endif
+
+
+//----------------------------------------------------------------
+// Thread init/done
+//----------------------------------------------------------------
+
+#if defined(MI_USE_PTHREADS)
+
+// use pthread local storage keys to detect thread ending
+// (and used with MI_TLS_PTHREADS for the default theap)
+pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
+
+static void mi_pthread_done(void* value) {
+  if (value!=NULL) {
+    _mi_thread_done((mi_theap_t*)value);
+  }
+}
+
+void _mi_prim_thread_init_auto_done(void) {
+  mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));
+  pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);
+}
+
+void _mi_prim_thread_done_auto_done(void) {
+  if (_mi_heap_default_key != (pthread_key_t)(-1)) {  // do not leak the key, see issue #809
+    pthread_key_delete(_mi_heap_default_key);
+  }
+}
+
+void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+  if (_mi_heap_default_key != (pthread_key_t)(-1)) {  // can happen during recursive invocation on freeBSD
+    pthread_setspecific(_mi_heap_default_key, theap);
+  }
+}
+
+#else
+
+void _mi_prim_thread_init_auto_done(void) {
+  // nothing
+}
+
+void _mi_prim_thread_done_auto_done(void) {
+  // nothing
+}
+
+void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+  MI_UNUSED(theap);
+}
+
+#endif
+
+bool _mi_prim_thread_is_in_threadpool(void) {
+  return false;
+}
+
+void _mi_prim_thread_yield(void) {
+  sleep(0);
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/wasi/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/wasi/prim.c
new file mode 100644
index 00000000..790daff0
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/wasi/prim.c
@@ -0,0 +1,291 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// This file is included in `src/prim/prim.c`
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"
+
+#include <stdio.h>   // fputs
+#include <stdlib.h>  // getenv
+#include <unistd.h>  // sbrk, sleep
+
+//---------------------------------------------
+// Initialize
+//---------------------------------------------
+
+void _mi_prim_mem_init( mi_os_mem_config_t* config ) {
+  config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
+  config->alloc_granularity = 16;
+  config->has_overcommit = false;
+  config->has_partial_free = false;
+  config->has_virtual_reserve = false;
+}
+
+//---------------------------------------------
+// Free
+//---------------------------------------------
+
+int _mi_prim_free(void* addr, size_t size ) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  // wasi theap cannot be shrunk
+  return 0;
+}
+
+
+//---------------------------------------------
+// Allocation: sbrk or memory_grow
+//---------------------------------------------
+
+#if defined(MI_USE_SBRK)
+  static void* mi_memory_grow( size_t size ) {
+    void* p = sbrk(size);
+    if (p == (void*)(-1)) return NULL;
+    #if !defined(__wasi__) // on wasi this is always zero initialized already (?)
+    memset(p,0,size);
+    #endif
+    return p;
+  }
+#elif defined(__wasi__)
+  static void* mi_memory_grow( size_t size ) {
+    size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size()))
+                            : __builtin_wasm_memory_size(0));
+    if (base == SIZE_MAX) return NULL;
+    return (void*)(base * _mi_os_page_size());
+  }
+#endif
+
+#if defined(MI_USE_PTHREADS)
+static pthread_mutex_t mi_theap_grow_mutex = PTHREAD_MUTEX_INITIALIZER;
+#endif
+
+static void* mi_prim_mem_grow(size_t size, size_t try_alignment) {
+  void* p = NULL;
+  if (try_alignment <= 1) {
+    // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now)
+    #if defined(MI_USE_PTHREADS)
+    pthread_mutex_lock(&mi_theap_grow_mutex);
+    #endif
+    p = mi_memory_grow(size);
+    #if defined(MI_USE_PTHREADS)
+    pthread_mutex_unlock(&mi_theap_grow_mutex);
+    #endif
+  }
+  else {
+    void* base = NULL;
+    size_t alloc_size = 0;
+    // to allocate aligned use a lock to try to avoid thread interaction
+    // between getting the current size and actual allocation
+    // (also, `sbrk` is not thread safe in general)
+    #if defined(MI_USE_PTHREADS)
+    pthread_mutex_lock(&mi_theap_grow_mutex);
+    #endif
+    {
+      void* current = mi_memory_grow(0);  // get current size
+      if (current != NULL) {
+        void* aligned_current = _mi_align_up_ptr(current, try_alignment);  // and align from there to minimize wasted space
+        alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size());
+        base = mi_memory_grow(alloc_size);
+      }
+    }
+    #if defined(MI_USE_PTHREADS)
+    pthread_mutex_unlock(&mi_theap_grow_mutex);
+    #endif
+    if (base != NULL) {
+      p = _mi_align_up_ptr(base, try_alignment);
+      if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) {
+        // another thread used wasm_memory_grow/sbrk in-between and we do not have enough
+        // space after alignment. Give up (and waste the space as we cannot shrink :-( )
+        // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align)
+        p = NULL;
+      }
+    }
+  }
+  /*
+  if (p == NULL) {
+    _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment);
+    errno = ENOMEM;
+    return NULL;
+  }
+  */
+  mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 );
+  return p;
+}
+
+// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
+int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
+  MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);
+  *is_large = false;
+  *is_zero = false;
+  *addr = mi_prim_mem_grow(size, try_alignment);
+  return (*addr != NULL ? 0 : ENOMEM);
+}
+
+
+//---------------------------------------------
+// Commit/Reset/Protect
+//---------------------------------------------
+
+int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  *is_zero = false;
+  return 0;
+}
+
+int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  *needs_recommit = false;
+  return 0;
+}
+
+int _mi_prim_reset(void* addr, size_t size) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  return 0;
+}
+
+int _mi_prim_reuse(void* addr, size_t size) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  return 0;
+}
+
+int _mi_prim_protect(void* addr, size_t size, bool protect) {
+  MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);
+  return 0;
+}
+
+
+//---------------------------------------------
+// Huge pages and NUMA nodes
+//---------------------------------------------
+
+int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
+  MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
+  *is_zero = true;
+  *addr = NULL;
+  return ENOSYS;
+}
+
+size_t _mi_prim_numa_node(void) {
+  return 0;
+}
+
+size_t _mi_prim_numa_node_count(void) {
+  return 1;
+}
+
+
+//----------------------------------------------------------------
+// Clock
+//----------------------------------------------------------------
+
+#include <time.h>
+
+#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
+
+mi_msecs_t _mi_prim_clock_now(void) {
+  struct timespec t;
+  #ifdef CLOCK_MONOTONIC
+  clock_gettime(CLOCK_MONOTONIC, &t);
+  #else
+  clock_gettime(CLOCK_REALTIME, &t);
+  #endif
+  return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);
+}
+
+#else
+
+// low resolution timer
+mi_msecs_t _mi_prim_clock_now(void) {
+  #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)
+  return (mi_msecs_t)clock();
+  #elif (CLOCKS_PER_SEC < 1000)
+  return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
+  #else
+  return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
+  #endif
+}
+
+#endif
+
+
+//----------------------------------------------------------------
+// Process info
+//----------------------------------------------------------------
+
+void _mi_prim_process_info(mi_process_info_t* pinfo)
+{
+  // use defaults
+  MI_UNUSED(pinfo);
+}
+
+
+//----------------------------------------------------------------
+// Output
+//----------------------------------------------------------------
+
+void _mi_prim_out_stderr( const char* msg ) {
+  fputs(msg,stderr);
+}
+
+
+//----------------------------------------------------------------
+// Environment
+//----------------------------------------------------------------
+
+bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+  // cannot call getenv() when still initializing the C runtime.
+  if (_mi_preloading()) return false;
+  const char* s = getenv(name);
+  if (s == NULL) {
+    // we check the upper case name too.
+    char buf[64+1];
+    size_t len = _mi_strnlen(name,sizeof(buf)-1);
+    for (size_t i = 0; i < len; i++) {
+      buf[i] = _mi_toupper(name[i]);
+    }
+    buf[len] = 0;
+    s = getenv(buf);
+  }
+  if (s == NULL || _mi_strnlen(s,result_size) >= result_size)  return false;
+  _mi_strlcpy(result, s, result_size);
+  return true;
+}
+
+
+//----------------------------------------------------------------
+// Random
+//----------------------------------------------------------------
+
+bool _mi_prim_random_buf(void* buf, size_t buf_len) {
+  return false;
+}
+
+
+//----------------------------------------------------------------
+// Thread init/done
+//----------------------------------------------------------------
+
+void _mi_prim_thread_init_auto_done(void) {
+  // nothing
+}
+
+void _mi_prim_thread_done_auto_done(void) {
+  // nothing
+}
+
+void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+  MI_UNUSED(theap);
+}
+
+bool _mi_prim_thread_is_in_threadpool(void) {
+  return false;
+}
+
+void _mi_prim_thread_yield(void) {
+  sleep(0);
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw-mimalloc.wprp b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw-mimalloc.wprp
new file mode 100644
index 00000000..b00cd7ad
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw-mimalloc.wprp
@@ -0,0 +1,61 @@
+<WindowsPerformanceRecorder Version="1.0">
+  <Profiles>
+    <SystemCollector Id="WPR_initiated_WprApp_WPR_System_Collector" Name="WPR_initiated_WprApp_WPR System Collector">
+      <BufferSize Value="1024" />
+      <Buffers Value="100" />
+    </SystemCollector>
+    <EventCollector Id="Mimalloc_Collector" Name="Mimalloc Collector">
+      <BufferSize Value="1024" />
+      <Buffers Value="100" />
+    </EventCollector>
+    <SystemProvider Id="WPR_initiated_WprApp_WPR_System_Collector_Provider">
+      <Keywords>
+        <Keyword Value="Loader" />
+      </Keywords>
+    </SystemProvider>
+    <EventProvider Id="MimallocEventProvider" Name="138f4dbb-ee04-4899-aa0a-572ad4475779" NonPagedMemory="true" Stack="true">
+      <EventFilters FilterIn="true">
+        <EventId Value="100" />
+        <EventId Value="101" />
+      </EventFilters>
+    </EventProvider>
+    <Profile Id="CustomHeap.Verbose.File" Name="CustomHeap" Description="RunningProfile:CustomHeap.Verbose.File" LoggingMode="File" DetailLevel="Verbose">
+      <ProblemCategories>
+        <ProblemCategory Value="Resource Analysis" />
+      </ProblemCategories>
+      <Collectors>
+        <SystemCollectorId Value="WPR_initiated_WprApp_WPR_System_Collector">
+          <SystemProviderId Value="WPR_initiated_WprApp_WPR_System_Collector_Provider" />
+        </SystemCollectorId>
+        <EventCollectorId Value="Mimalloc_Collector">
+          <EventProviders>
+            <EventProviderId Value="MimallocEventProvider" >
+              <Keywords>
+                <Keyword Value="100"/>
+                <Keyword Value="101"/>
+              </Keywords>
+            </EventProviderId>
+          </EventProviders>
+        </EventCollectorId>
+      </Collectors>
+      <TraceMergeProperties>
+        <TraceMergeProperty Id="BaseVerboseTraceMergeProperties" Name="BaseTraceMergeProperties">
+          <DeletePreMergedTraceFiles Value="true" />
+          <FileCompression Value="false" />
+          <InjectOnly Value="false" />
+          <SkipMerge Value="false" />
+          <CustomEvents>
+            <CustomEvent Value="ImageId" />
+            <CustomEvent Value="BuildInfo" />
+            <CustomEvent Value="VolumeMapping" />
+            <CustomEvent Value="EventMetadata" />
+            <CustomEvent Value="PerfTrackMetadata" />
+            <CustomEvent Value="WinSAT" />
+            <CustomEvent Value="NetworkInterface" />
+          </CustomEvents>
+        </TraceMergeProperty>
+      </TraceMergeProperties>
+    </Profile>
+  </Profiles>
+</WindowsPerformanceRecorder>
+
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.h
new file mode 100644
index 00000000..4e0a092a
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.h
@@ -0,0 +1,905 @@
+//**********************************************************************`
+//* This is an include file generated by Message Compiler.             *`
+//*                                                                    *`
+//* Copyright (c) Microsoft Corporation. All Rights Reserved.          *`
+//**********************************************************************`
+#pragma once
+
+//*****************************************************************************
+//
+// Notes on the ETW event code generated by MC:
+//
+// - Structures and arrays of structures are treated as an opaque binary blob.
+//   The caller is responsible for packing the data for the structure into a
+//   single region of memory, with no padding between values. The macro will
+//   have an extra parameter for the length of the blob.
+// - Arrays of nul-terminated strings must be packed by the caller into a
+//   single binary blob containing the correct number of strings, with a nul
+//   after each string. The size of the blob is specified in characters, and
+//   includes the final nul.
+// - Arrays of SID are treated as a single binary blob. The caller is
+//   responsible for packing the SID values into a single region of memory with
+//   no padding.
+// - The length attribute on the data element in the manifest is significant
+//   for values with intype win:UnicodeString, win:AnsiString, or win:Binary.
+//   The length attribute must be specified for win:Binary, and is optional for
+//   win:UnicodeString and win:AnsiString (if no length is given, the strings
+//   are assumed to be nul-terminated). For win:UnicodeString, the length is
+//   measured in characters, not bytes.
+// - For an array of win:UnicodeString, win:AnsiString, or win:Binary, the
+//   length attribute applies to every value in the array, so every value in
+//   the array must have the same length. The values in the array are provided
+//   to the macro via a single pointer -- the caller is responsible for packing
+//   all of the values into a single region of memory with no padding between
+//   values.
+// - Values of type win:CountedUnicodeString, win:CountedAnsiString, and
+//   win:CountedBinary can be generated and collected on Vista or later.
+//   However, they may not decode properly without the Windows 10 2018 Fall
+//   Update.
+// - Arrays of type win:CountedUnicodeString, win:CountedAnsiString, and
+//   win:CountedBinary must be packed by the caller into a single region of
+//   memory. The format for each item is a UINT16 byte-count followed by that
+//   many bytes of data. When providing the array to the generated macro, you
+//   must provide the total size of the packed array data, including the UINT16
+//   sizes for each item. In the case of win:CountedUnicodeString, the data
+//   size is specified in WCHAR (16-bit) units. In the case of
+//   win:CountedAnsiString and win:CountedBinary, the data size is specified in
+//   bytes.
+//
+//*****************************************************************************
+
+#include <wmistr.h>
+#include <evntrace.h>
+#include <evntprov.h>
+
+#ifndef ETW_INLINE
+  #ifdef _ETW_KM_
+    // In kernel mode, save stack space by never inlining templates.
+    #define ETW_INLINE DECLSPEC_NOINLINE __inline
+  #else
+    // In user mode, save code size by inlining templates as appropriate.
+    #define ETW_INLINE __inline
+  #endif
+#endif // ETW_INLINE
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+//
+// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:
+// Define this macro to have the compiler skip the generated functions in this
+// header.
+//
+#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION
+
+//
+// MCGEN_USE_KERNEL_MODE_APIS macro:
+// Controls whether the generated code uses kernel-mode or user-mode APIs.
+// - Set to 0 to use Windows user-mode APIs such as EventRegister.
+// - Set to 1 to use Windows kernel-mode APIs such as EtwRegister.
+// Default is based on whether the _ETW_KM_ macro is defined (i.e. by wdm.h).
+// Note that the APIs can also be overridden directly, e.g. by setting the
+// MCGEN_EVENTWRITETRANSFER or MCGEN_EVENTREGISTER macros.
+//
+#ifndef MCGEN_USE_KERNEL_MODE_APIS
+  #ifdef _ETW_KM_
+    #define MCGEN_USE_KERNEL_MODE_APIS 1
+  #else
+    #define MCGEN_USE_KERNEL_MODE_APIS 0
+  #endif
+#endif // MCGEN_USE_KERNEL_MODE_APIS
+
+//
+// MCGEN_HAVE_EVENTSETINFORMATION macro:
+// Controls how McGenEventSetInformation uses the EventSetInformation API.
+// - Set to 0 to disable the use of EventSetInformation
+//   (McGenEventSetInformation will always return an error).
+// - Set to 1 to directly invoke MCGEN_EVENTSETINFORMATION.
+// - Set to 2 to to locate EventSetInformation at runtime via GetProcAddress
+//   (user-mode) or MmGetSystemRoutineAddress (kernel-mode).
+// Default is determined as follows:
+// - If MCGEN_EVENTSETINFORMATION has been customized, set to 1
+//   (i.e. use MCGEN_EVENTSETINFORMATION).
+// - Else if the target OS version has EventSetInformation, set to 1
+//   (i.e. use MCGEN_EVENTSETINFORMATION).
+// - Else set to 2 (i.e. try to dynamically locate EventSetInformation).
+// Note that an McGenEventSetInformation function will only be generated if one
+// or more provider in a manifest has provider traits.
+//
+#ifndef MCGEN_HAVE_EVENTSETINFORMATION
+  #ifdef MCGEN_EVENTSETINFORMATION             // if MCGEN_EVENTSETINFORMATION has been customized,
+    #define MCGEN_HAVE_EVENTSETINFORMATION   1 //   directly invoke MCGEN_EVENTSETINFORMATION(...).
+  #elif MCGEN_USE_KERNEL_MODE_APIS             // else if using kernel-mode APIs,
+    #if NTDDI_VERSION >= 0x06040000            //   if target OS is Windows 10 or later,
+      #define MCGEN_HAVE_EVENTSETINFORMATION 1 //     directly invoke MCGEN_EVENTSETINFORMATION(...).
+    #else                                      //   else
+      #define MCGEN_HAVE_EVENTSETINFORMATION 2 //     find "EtwSetInformation" via MmGetSystemRoutineAddress.
+    #endif                                     // else (using user-mode APIs)
+  #else                                        //   if target OS and SDK is Windows 8 or later,
+    #if WINVER >= 0x0602 && defined(EVENT_FILTER_TYPE_SCHEMATIZED)
+      #define MCGEN_HAVE_EVENTSETINFORMATION 1 //     directly invoke MCGEN_EVENTSETINFORMATION(...).
+    #else                                      //   else
+      #define MCGEN_HAVE_EVENTSETINFORMATION 2 //     find "EventSetInformation" via GetModuleHandleExW/GetProcAddress.
+    #endif
+  #endif
+#endif // MCGEN_HAVE_EVENTSETINFORMATION
+
+//
+// MCGEN Override Macros
+//
+// The following override macros may be defined before including this header
+// to control the APIs used by this header:
+//
+// - MCGEN_EVENTREGISTER
+// - MCGEN_EVENTUNREGISTER
+// - MCGEN_EVENTSETINFORMATION
+// - MCGEN_EVENTWRITETRANSFER
+//
+// If the the macro is undefined, the MC implementation will default to the
+// corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is
+// undefined, the EventRegister[MyProviderName] macro will use EventRegister
+// in user mode and will use EtwRegister in kernel mode.
+//
+// To prevent issues from conflicting definitions of these macros, the value
+// of the override macro will be used as a suffix in certain internal function
+// names. Because of this, the override macros must follow certain rules:
+//
+// - The macro must be defined before any MC-generated header is included and
+//   must not be undefined or redefined after any MC-generated header is
+//   included. Different translation units (i.e. different .c or .cpp files)
+//   may set the macros to different values, but within a translation unit
+//   (within a single .c or .cpp file), the macro must be set once and not
+//   changed.
+// - The override must be an object-like macro, not a function-like macro
+//   (i.e. the override macro must not have a parameter list).
+// - The override macro's value must be a simple identifier, i.e. must be
+//   something that starts with a letter or '_' and contains only letters,
+//   numbers, and '_' characters.
+// - If the override macro's value is the name of a second object-like macro,
+//   the second object-like macro must follow the same rules. (The override
+//   macro's value can also be the name of a function-like macro, in which
+//   case the function-like macro does not need to follow the same rules.)
+//
+// For example, the following will cause compile errors:
+//
+//   #define MCGEN_EVENTWRITETRANSFER MyNamespace::MyClass::MyFunction // Value has non-identifier characters (colon).
+//   #define MCGEN_EVENTWRITETRANSFER GetEventWriteFunctionPointer(7)  // Value has non-identifier characters (parentheses).
+//   #define MCGEN_EVENTWRITETRANSFER(h,e,a,r,c,d) EventWrite(h,e,c,d) // Override is defined as a function-like macro.
+//   #define MY_OBJECT_LIKE_MACRO     MyNamespace::MyClass::MyEventWriteFunction
+//   #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // Evaluates to something with non-identifier characters (colon).
+//
+// The following would be ok:
+//
+//   #define MCGEN_EVENTWRITETRANSFER  MyEventWriteFunction1  // OK, suffix will be "MyEventWriteFunction1".
+//   #define MY_OBJECT_LIKE_MACRO      MyEventWriteFunction2
+//   #define MCGEN_EVENTWRITETRANSFER  MY_OBJECT_LIKE_MACRO   // OK, suffix will be "MyEventWriteFunction2".
+//   #define MY_FUNCTION_LIKE_MACRO(h,e,a,r,c,d) MyNamespace::MyClass::MyEventWriteFunction3(h,e,c,d)
+//   #define MCGEN_EVENTWRITETRANSFER  MY_FUNCTION_LIKE_MACRO // OK, suffix will be "MY_FUNCTION_LIKE_MACRO".
+//
+#ifndef MCGEN_EVENTREGISTER
+  #if MCGEN_USE_KERNEL_MODE_APIS
+    #define MCGEN_EVENTREGISTER        EtwRegister
+  #else
+    #define MCGEN_EVENTREGISTER        EventRegister
+  #endif
+#endif // MCGEN_EVENTREGISTER
+#ifndef MCGEN_EVENTUNREGISTER
+  #if MCGEN_USE_KERNEL_MODE_APIS
+    #define MCGEN_EVENTUNREGISTER      EtwUnregister
+  #else
+    #define MCGEN_EVENTUNREGISTER      EventUnregister
+  #endif
+#endif // MCGEN_EVENTUNREGISTER
+#ifndef MCGEN_EVENTSETINFORMATION
+  #if MCGEN_USE_KERNEL_MODE_APIS
+    #define MCGEN_EVENTSETINFORMATION  EtwSetInformation
+  #else
+    #define MCGEN_EVENTSETINFORMATION  EventSetInformation
+  #endif
+#endif // MCGEN_EVENTSETINFORMATION
+#ifndef MCGEN_EVENTWRITETRANSFER
+  #if MCGEN_USE_KERNEL_MODE_APIS
+    #define MCGEN_EVENTWRITETRANSFER   EtwWriteTransfer
+  #else
+    #define MCGEN_EVENTWRITETRANSFER   EventWriteTransfer
+  #endif
+#endif // MCGEN_EVENTWRITETRANSFER
+
+//
+// MCGEN_EVENT_ENABLED macro:
+// Override to control how the EventWrite[EventName] macros determine whether
+// an event is enabled. The default behavior is for EventWrite[EventName] to
+// use the EventEnabled[EventName] macros.
+//
+#ifndef MCGEN_EVENT_ENABLED
+#define MCGEN_EVENT_ENABLED(EventName) EventEnabled##EventName()
+#endif
+
+//
+// MCGEN_EVENT_ENABLED_FORCONTEXT macro:
+// Override to control how the EventWrite[EventName]_ForContext macros
+// determine whether an event is enabled. The default behavior is for
+// EventWrite[EventName]_ForContext to use the
+// EventEnabled[EventName]_ForContext macros.
+//
+#ifndef MCGEN_EVENT_ENABLED_FORCONTEXT
+#define MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, EventName) EventEnabled##EventName##_ForContext(pContext)
+#endif
+
+//
+// MCGEN_ENABLE_CHECK macro:
+// Determines whether the specified event would be considered as enabled
+// based on the state of the specified context. Slightly faster than calling
+// McGenEventEnabled directly.
+//
+#ifndef MCGEN_ENABLE_CHECK
+#define MCGEN_ENABLE_CHECK(Context, Descriptor) (Context.IsEnabled && McGenEventEnabled(&Context, &Descriptor))
+#endif
+
+#if !defined(MCGEN_TRACE_CONTEXT_DEF)
+#define MCGEN_TRACE_CONTEXT_DEF
+// This structure is for use by MC-generated code and should not be used directly.
+typedef struct _MCGEN_TRACE_CONTEXT
+{
+    TRACEHANDLE            RegistrationHandle;
+    TRACEHANDLE            Logger;      // Used as pointer to provider traits.
+    ULONGLONG              MatchAnyKeyword;
+    ULONGLONG              MatchAllKeyword;
+    ULONG                  Flags;
+    ULONG                  IsEnabled;
+    UCHAR                  Level;
+    UCHAR                  Reserve;
+    USHORT                 EnableBitsCount;
+    PULONG                 EnableBitMask;
+    const ULONGLONG*       EnableKeyWords;
+    const UCHAR*           EnableLevel;
+} MCGEN_TRACE_CONTEXT, *PMCGEN_TRACE_CONTEXT;
+#endif // MCGEN_TRACE_CONTEXT_DEF
+
+#if !defined(MCGEN_LEVEL_KEYWORD_ENABLED_DEF)
+#define MCGEN_LEVEL_KEYWORD_ENABLED_DEF
+//
+// Determines whether an event with a given Level and Keyword would be
+// considered as enabled based on the state of the specified context.
+// Note that you may want to use MCGEN_ENABLE_CHECK instead of calling this
+// function directly.
+//
+FORCEINLINE
+BOOLEAN
+McGenLevelKeywordEnabled(
+    _In_ PMCGEN_TRACE_CONTEXT EnableInfo,
+    _In_ UCHAR Level,
+    _In_ ULONGLONG Keyword
+    )
+{
+    //
+    // Check if the event Level is lower than the level at which
+    // the channel is enabled.
+    // If the event Level is 0 or the channel is enabled at level 0,
+    // all levels are enabled.
+    //
+
+    if ((Level <= EnableInfo->Level) || // This also covers the case of Level == 0.
+        (EnableInfo->Level == 0)) {
+
+        //
+        // Check if Keyword is enabled
+        //
+
+        if ((Keyword == (ULONGLONG)0) ||
+            ((Keyword & EnableInfo->MatchAnyKeyword) &&
+             ((Keyword & EnableInfo->MatchAllKeyword) == EnableInfo->MatchAllKeyword))) {
+            return TRUE;
+        }
+    }
+
+    return FALSE;
+}
+#endif // MCGEN_LEVEL_KEYWORD_ENABLED_DEF
+
+#if !defined(MCGEN_EVENT_ENABLED_DEF)
+#define MCGEN_EVENT_ENABLED_DEF
+//
+// Determines whether the specified event would be considered as enabled based
+// on the state of the specified context. Note that you may want to use
+// MCGEN_ENABLE_CHECK instead of calling this function directly.
+//
+FORCEINLINE
+BOOLEAN
+McGenEventEnabled(
+    _In_ PMCGEN_TRACE_CONTEXT EnableInfo,
+    _In_ PCEVENT_DESCRIPTOR EventDescriptor
+    )
+{
+    return McGenLevelKeywordEnabled(EnableInfo, EventDescriptor->Level, EventDescriptor->Keyword);
+}
+#endif // MCGEN_EVENT_ENABLED_DEF
+
+#if !defined(MCGEN_CONTROL_CALLBACK)
+#define MCGEN_CONTROL_CALLBACK
+
+// This function is for use by MC-generated code and should not be used directly.
+DECLSPEC_NOINLINE __inline
+VOID
+__stdcall
+McGenControlCallbackV2(
+    _In_ LPCGUID SourceId,
+    _In_ ULONG ControlCode,
+    _In_ UCHAR Level,
+    _In_ ULONGLONG MatchAnyKeyword,
+    _In_ ULONGLONG MatchAllKeyword,
+    _In_opt_ PEVENT_FILTER_DESCRIPTOR FilterData,
+    _Inout_opt_ PVOID CallbackContext
+    )
+/*++
+
+Routine Description:
+
+    This is the notification callback for Windows Vista and later.
+
+Arguments:
+
+    SourceId - The GUID that identifies the session that enabled the provider.
+
+    ControlCode - The parameter indicates whether the provider
+                  is being enabled or disabled.
+
+    Level - The level at which the event is enabled.
+
+    MatchAnyKeyword - The bitmask of keywords that the provider uses to
+                      determine the category of events that it writes.
+
+    MatchAllKeyword - This bitmask additionally restricts the category
+                      of events that the provider writes.
+
+    FilterData - The provider-defined data.
+
+    CallbackContext - The context of the callback that is defined when the provider
+                      called EtwRegister to register itself.
+
+Remarks:
+
+    ETW calls this function to notify provider of enable/disable
+
+--*/
+{
+    PMCGEN_TRACE_CONTEXT Ctx = (PMCGEN_TRACE_CONTEXT)CallbackContext;
+    ULONG Ix;
+#ifndef MCGEN_PRIVATE_ENABLE_CALLBACK_V2
+    UNREFERENCED_PARAMETER(SourceId);
+    UNREFERENCED_PARAMETER(FilterData);
+#endif
+
+    if (Ctx == NULL) {
+        return;
+    }
+
+    switch (ControlCode) {
+
+        case EVENT_CONTROL_CODE_ENABLE_PROVIDER:
+            Ctx->Level = Level;
+            Ctx->MatchAnyKeyword = MatchAnyKeyword;
+            Ctx->MatchAllKeyword = MatchAllKeyword;
+            Ctx->IsEnabled = EVENT_CONTROL_CODE_ENABLE_PROVIDER;
+
+            for (Ix = 0; Ix < Ctx->EnableBitsCount; Ix += 1) {
+                if (McGenLevelKeywordEnabled(Ctx, Ctx->EnableLevel[Ix], Ctx->EnableKeyWords[Ix]) != FALSE) {
+                    Ctx->EnableBitMask[Ix >> 5] |= (1 << (Ix % 32));
+                } else {
+                    Ctx->EnableBitMask[Ix >> 5] &= ~(1 << (Ix % 32));
+                }
+            }
+            break;
+
+        case EVENT_CONTROL_CODE_DISABLE_PROVIDER:
+            Ctx->IsEnabled = EVENT_CONTROL_CODE_DISABLE_PROVIDER;
+            Ctx->Level = 0;
+            Ctx->MatchAnyKeyword = 0;
+            Ctx->MatchAllKeyword = 0;
+            if (Ctx->EnableBitsCount > 0) {
+#pragma warning(suppress: 26451) // Arithmetic overflow cannot occur, no matter the value of EnableBitCount
+                RtlZeroMemory(Ctx->EnableBitMask, (((Ctx->EnableBitsCount - 1) / 32) + 1) * sizeof(ULONG));
+            }
+            break;
+
+        default:
+            break;
+    }
+
+#ifdef MCGEN_PRIVATE_ENABLE_CALLBACK_V2
+    //
+    // Call user defined callback
+    //
+    MCGEN_PRIVATE_ENABLE_CALLBACK_V2(
+        SourceId,
+        ControlCode,
+        Level,
+        MatchAnyKeyword,
+        MatchAllKeyword,
+        FilterData,
+        CallbackContext
+        );
+#endif // MCGEN_PRIVATE_ENABLE_CALLBACK_V2
+
+    return;
+}
+
+#endif // MCGEN_CONTROL_CALLBACK
+
+#ifndef _mcgen_PENABLECALLBACK
+  #if MCGEN_USE_KERNEL_MODE_APIS
+    #define _mcgen_PENABLECALLBACK      PETWENABLECALLBACK
+  #else
+    #define _mcgen_PENABLECALLBACK      PENABLECALLBACK
+  #endif
+#endif // _mcgen_PENABLECALLBACK
+
+#if !defined(_mcgen_PASTE2)
+// This macro is for use by MC-generated code and should not be used directly.
+#define _mcgen_PASTE2(a, b) _mcgen_PASTE2_imp(a, b)
+#define _mcgen_PASTE2_imp(a, b) a##b
+#endif // _mcgen_PASTE2
+
+#if !defined(_mcgen_PASTE3)
+// This macro is for use by MC-generated code and should not be used directly.
+#define _mcgen_PASTE3(a, b, c) _mcgen_PASTE3_imp(a, b, c)
+#define _mcgen_PASTE3_imp(a, b, c) a##b##_##c
+#endif // _mcgen_PASTE3
+
+//
+// Macro validation
+//
+
+// Validate MCGEN_EVENTREGISTER:
+
+// Trigger an error if MCGEN_EVENTREGISTER is not an unqualified (simple) identifier:
+struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER);
+
+// Trigger an error if MCGEN_EVENTREGISTER is redefined:
+typedef struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER)
+    MCGEN_EVENTREGISTER_must_not_be_redefined_between_headers;
+
+// Trigger an error if MCGEN_EVENTREGISTER is defined as a function-like macro:
+typedef void MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTREGISTER;
+typedef int _mcgen_PASTE2(MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTREGISTER);
+
+// Validate MCGEN_EVENTUNREGISTER:
+
+// Trigger an error if MCGEN_EVENTUNREGISTER is not an unqualified (simple) identifier:
+struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER);
+
+// Trigger an error if MCGEN_EVENTUNREGISTER is redefined:
+typedef struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER)
+    MCGEN_EVENTUNREGISTER_must_not_be_redefined_between_headers;
+
+// Trigger an error if MCGEN_EVENTUNREGISTER is defined as a function-like macro:
+typedef void MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTUNREGISTER;
+typedef int _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTUNREGISTER);
+
+// Validate MCGEN_EVENTSETINFORMATION:
+
+// Trigger an error if MCGEN_EVENTSETINFORMATION is not an unqualified (simple) identifier:
+struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION);
+
+// Trigger an error if MCGEN_EVENTSETINFORMATION is redefined:
+typedef struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION)
+    MCGEN_EVENTSETINFORMATION_must_not_be_redefined_between_headers;
+
+// Trigger an error if MCGEN_EVENTSETINFORMATION is defined as a function-like macro:
+typedef void MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_MCGEN_EVENTSETINFORMATION;
+typedef int _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_, MCGEN_EVENTSETINFORMATION);
+
+// Validate MCGEN_EVENTWRITETRANSFER:
+
+// Trigger an error if MCGEN_EVENTWRITETRANSFER is not an unqualified (simple) identifier:
+struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER);
+
+// Trigger an error if MCGEN_EVENTWRITETRANSFER is redefined:
+typedef struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER)
+    MCGEN_EVENTWRITETRANSFER_must_not_be_redefined_between_headers;;
+
+// Trigger an error if MCGEN_EVENTWRITETRANSFER is defined as a function-like macro:
+typedef void MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_MCGEN_EVENTWRITETRANSFER;
+typedef int _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_, MCGEN_EVENTWRITETRANSFER);
+
+#ifndef McGenEventWrite_def
+#define McGenEventWrite_def
+
+// This macro is for use by MC-generated code and should not be used directly.
+#define McGenEventWrite _mcgen_PASTE2(McGenEventWrite_, MCGEN_EVENTWRITETRANSFER)
+
+// This function is for use by MC-generated code and should not be used directly.
+DECLSPEC_NOINLINE __inline
+ULONG __stdcall
+McGenEventWrite(
+    _In_ PMCGEN_TRACE_CONTEXT Context,
+    _In_ PCEVENT_DESCRIPTOR Descriptor,
+    _In_opt_ LPCGUID ActivityId,
+    _In_range_(1, 128) ULONG EventDataCount,
+    _Pre_cap_(EventDataCount) EVENT_DATA_DESCRIPTOR* EventData
+    )
+{
+    const USHORT UNALIGNED* Traits;
+
+    // Some customized MCGEN_EVENTWRITETRANSFER macros might ignore ActivityId.
+    UNREFERENCED_PARAMETER(ActivityId);
+
+    Traits = (const USHORT UNALIGNED*)(UINT_PTR)Context->Logger;
+
+    if (Traits == NULL) {
+        EventData[0].Ptr = 0;
+        EventData[0].Size = 0;
+        EventData[0].Reserved = 0;
+    } else {
+        EventData[0].Ptr = (ULONG_PTR)Traits;
+        EventData[0].Size = *Traits;
+        EventData[0].Reserved = 2; // EVENT_DATA_DESCRIPTOR_TYPE_PROVIDER_METADATA
+    }
+
+    return MCGEN_EVENTWRITETRANSFER(
+        Context->RegistrationHandle,
+        Descriptor,
+        ActivityId,
+        NULL,
+        EventDataCount,
+        EventData);
+}
+#endif // McGenEventWrite_def
+
+#if !defined(McGenEventRegisterUnregister)
+#define McGenEventRegisterUnregister
+
+// This macro is for use by MC-generated code and should not be used directly.
+#define McGenEventRegister _mcgen_PASTE2(McGenEventRegister_, MCGEN_EVENTREGISTER)
+
+#pragma warning(push)
+#pragma warning(disable:6103)
+// This function is for use by MC-generated code and should not be used directly.
+DECLSPEC_NOINLINE __inline
+ULONG __stdcall
+McGenEventRegister(
+    _In_ LPCGUID ProviderId,
+    _In_opt_ _mcgen_PENABLECALLBACK EnableCallback,
+    _In_opt_ PVOID CallbackContext,
+    _Inout_ PREGHANDLE RegHandle
+    )
+/*++
+
+Routine Description:
+
+    This function registers the provider with ETW.
+
+Arguments:
+
+    ProviderId - Provider ID to register with ETW.
+
+    EnableCallback - Callback to be used.
+
+    CallbackContext - Context for the callback.
+
+    RegHandle - Pointer to registration handle.
+
+Remarks:
+
+    Should not be called if the provider is already registered (i.e. should not
+    be called if *RegHandle != 0). Repeatedly registering a provider is a bug
+    and may indicate a race condition. However, for compatibility with previous
+    behavior, this function will return SUCCESS in this case.
+
+--*/
+{
+    ULONG Error;
+
+    if (*RegHandle != 0)
+    {
+        Error = 0; // ERROR_SUCCESS
+    }
+    else
+    {
+        Error = MCGEN_EVENTREGISTER(ProviderId, EnableCallback, CallbackContext, RegHandle);
+    }
+
+    return Error;
+}
+#pragma warning(pop)
+
+// This macro is for use by MC-generated code and should not be used directly.
+#define McGenEventUnregister _mcgen_PASTE2(McGenEventUnregister_, MCGEN_EVENTUNREGISTER)
+
+// This function is for use by MC-generated code and should not be used directly.
+DECLSPEC_NOINLINE __inline
+ULONG __stdcall
+McGenEventUnregister(_Inout_ PREGHANDLE RegHandle)
+/*++
+
+Routine Description:
+
+    Unregister from ETW and set *RegHandle = 0.
+
+Arguments:
+
+    RegHandle - the pointer to the provider registration handle
+
+Remarks:
+
+    If provider has not been registered (i.e. if *RegHandle == 0),
+    return SUCCESS. It is safe to call McGenEventUnregister even if the
+    call to McGenEventRegister returned an error.
+
+--*/
+{
+    ULONG Error;
+
+    if(*RegHandle == 0)
+    {
+        Error = 0; // ERROR_SUCCESS
+    }
+    else
+    {
+        Error = MCGEN_EVENTUNREGISTER(*RegHandle);
+        *RegHandle = (REGHANDLE)0;
+    }
+
+    return Error;
+}
+
+#endif // McGenEventRegisterUnregister
+
+#ifndef _mcgen_EVENT_BIT_SET
+  #if defined(_M_IX86) || defined(_M_X64)
+    // This macro is for use by MC-generated code and should not be used directly.
+    #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((((const unsigned char*)EnableBits)[BitPosition >> 3] & (1u << (BitPosition & 7))) != 0)
+  #else // CPU type
+    // This macro is for use by MC-generated code and should not be used directly.
+    #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((EnableBits[BitPosition >> 5] & (1u << (BitPosition & 31))) != 0)
+  #endif // CPU type
+#endif // _mcgen_EVENT_BIT_SET
+
+#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION
+
+//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+// Provider "microsoft-windows-mimalloc" event count 2
+//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+// Provider GUID = 138f4dbb-ee04-4899-aa0a-572ad4475779
+EXTERN_C __declspec(selectany) const GUID ETW_MI_Provider = {0x138f4dbb, 0xee04, 0x4899, {0xaa, 0x0a, 0x57, 0x2a, 0xd4, 0x47, 0x57, 0x79}};
+
+#ifndef ETW_MI_Provider_Traits
+#define ETW_MI_Provider_Traits NULL
+#endif // ETW_MI_Provider_Traits
+
+//
+// Event Descriptors
+//
+EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_ALLOC = {0x64, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};
+#define ETW_MI_ALLOC_value 0x64
+EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_FREE = {0x65, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};
+#define ETW_MI_FREE_value 0x65
+
+//
+// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:
+// Define this macro to have the compiler skip the generated functions in this
+// header.
+//
+#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION
+
+//
+// Event Enablement Bits
+// These variables are for use by MC-generated code and should not be used directly.
+//
+EXTERN_C __declspec(selectany) DECLSPEC_CACHEALIGN ULONG microsoft_windows_mimallocEnableBits[1];
+EXTERN_C __declspec(selectany) const ULONGLONG microsoft_windows_mimallocKeywords[1] = {0x0};
+EXTERN_C __declspec(selectany) const unsigned char microsoft_windows_mimallocLevels[1] = {4};
+
+//
+// Provider context
+//
+EXTERN_C __declspec(selectany) MCGEN_TRACE_CONTEXT ETW_MI_Provider_Context = {0, (ULONG_PTR)ETW_MI_Provider_Traits, 0, 0, 0, 0, 0, 0, 1, microsoft_windows_mimallocEnableBits, microsoft_windows_mimallocKeywords, microsoft_windows_mimallocLevels};
+
+//
+// Provider REGHANDLE
+//
+#define microsoft_windows_mimallocHandle (ETW_MI_Provider_Context.RegistrationHandle)
+
+//
+// This macro is set to 0, indicating that the EventWrite[Name] macros do not
+// have an Activity parameter. This is controlled by the -km and -um options.
+//
+#define ETW_MI_Provider_EventWriteActivity 0
+
+//
+// Register with ETW using the control GUID specified in the manifest.
+// Invoke this macro during module initialization (i.e. program startup,
+// DLL process attach, or driver load) to initialize the provider.
+// Note that if this function returns an error, the error means that
+// will not work, but no action needs to be taken -- even if EventRegister
+// returns an error, it is generally safe to use EventWrite and
+// EventUnregister macros (they will be no-ops if EventRegister failed).
+//
+#ifndef EventRegistermicrosoft_windows_mimalloc
+#define EventRegistermicrosoft_windows_mimalloc() McGenEventRegister(&ETW_MI_Provider, McGenControlCallbackV2, &ETW_MI_Provider_Context, &microsoft_windows_mimallocHandle)
+#endif
+
+//
+// Register with ETW using a specific control GUID (i.e. a GUID other than what
+// is specified in the manifest). Advanced scenarios only.
+//
+#ifndef EventRegisterByGuidmicrosoft_windows_mimalloc
+#define EventRegisterByGuidmicrosoft_windows_mimalloc(Guid) McGenEventRegister(&(Guid), McGenControlCallbackV2, &ETW_MI_Provider_Context, &microsoft_windows_mimallocHandle)
+#endif
+
+//
+// Unregister with ETW and close the provider.
+// Invoke this macro during module shutdown (i.e. program exit, DLL process
+// detach, or driver unload) to unregister the provider.
+// Note that you MUST call EventUnregister before DLL or driver unload
+// (not optional): failure to unregister a provider before DLL or driver unload
+// will result in crashes.
+//
+#ifndef EventUnregistermicrosoft_windows_mimalloc
+#define EventUnregistermicrosoft_windows_mimalloc() McGenEventUnregister(&microsoft_windows_mimallocHandle)
+#endif
+
+//
+// MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION macro:
+// Define this macro to enable support for caller-allocated provider context.
+//
+#ifdef MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION
+
+//
+// Advanced scenarios: Caller-allocated provider context.
+// Use when multiple differently-configured provider handles are needed,
+// e.g. for container-aware drivers, one context per container.
+//
+// Usage:
+//
+// - Caller enables the feature before including this header, e.g.
+//   #define MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION 1
+// - Caller allocates memory, e.g. pContext = malloc(sizeof(McGenContext_microsoft_windows_mimalloc));
+// - Caller registers the provider, e.g. EventRegistermicrosoft_windows_mimalloc_ForContext(pContext);
+// - Caller writes events, e.g. EventWriteMyEvent_ForContext(pContext, ...);
+// - Caller unregisters, e.g. EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext);
+// - Caller frees memory, e.g. free(pContext);
+//
+
+typedef struct tagMcGenContext_microsoft_windows_mimalloc {
+    // The fields of this structure are subject to change and should
+    // not be accessed directly. To access the provider's REGHANDLE,
+    // use microsoft_windows_mimallocHandle_ForContext(pContext).
+    MCGEN_TRACE_CONTEXT Context;
+    ULONG EnableBits[1];
+} McGenContext_microsoft_windows_mimalloc;
+
+#define EventRegistermicrosoft_windows_mimalloc_ForContext(pContext)             _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&ETW_MI_Provider, pContext)
+#define EventRegisterByGuidmicrosoft_windows_mimalloc_ForContext(Guid, pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&(Guid), pContext)
+#define EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext)           McGenEventUnregister(&(pContext)->Context.RegistrationHandle)
+
+//
+// Provider REGHANDLE for caller-allocated context.
+//
+#define microsoft_windows_mimallocHandle_ForContext(pContext) ((pContext)->Context.RegistrationHandle)
+
+// This function is for use by MC-generated code and should not be used directly.
+// Initialize and register the caller-allocated context.
+__inline
+ULONG __stdcall
+_mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(
+    _In_ LPCGUID pProviderId,
+    _Out_ McGenContext_microsoft_windows_mimalloc* pContext)
+{
+    RtlZeroMemory(pContext, sizeof(*pContext));
+    pContext->Context.Logger = (ULONG_PTR)ETW_MI_Provider_Traits;
+    pContext->Context.EnableBitsCount = 1;
+    pContext->Context.EnableBitMask = pContext->EnableBits;
+    pContext->Context.EnableKeyWords = microsoft_windows_mimallocKeywords;
+    pContext->Context.EnableLevel = microsoft_windows_mimallocLevels;
+    return McGenEventRegister(
+        pProviderId,
+        McGenControlCallbackV2,
+        &pContext->Context,
+        &pContext->Context.RegistrationHandle);
+}
+
+// This function is for use by MC-generated code and should not be used directly.
+// Trigger a compile error if called with the wrong parameter type.
+FORCEINLINE
+_Ret_ McGenContext_microsoft_windows_mimalloc*
+_mcgen_CheckContextType_microsoft_windows_mimalloc(_In_ McGenContext_microsoft_windows_mimalloc* pContext)
+{
+    return pContext;
+}
+
+#endif // MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION
+
+//
+// Enablement check macro for event "ETW_MI_ALLOC"
+//
+#define EventEnabledETW_MI_ALLOC() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)
+#define EventEnabledETW_MI_ALLOC_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)
+
+//
+// Event write macros for event "ETW_MI_ALLOC"
+//
+#define EventWriteETW_MI_ALLOC(Address, Size) \
+        MCGEN_EVENT_ENABLED(ETW_MI_ALLOC) \
+        ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) : 0
+#define EventWriteETW_MI_ALLOC_AssumeEnabled(Address, Size) \
+        _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size)
+#define EventWriteETW_MI_ALLOC_ForContext(pContext, Address, Size) \
+        MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_ALLOC) \
+        ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&(pContext)->Context, &ETW_MI_ALLOC, Address, Size) : 0
+#define EventWriteETW_MI_ALLOC_ForContextAssumeEnabled(pContext, Address, Size) \
+        _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_ALLOC, Address, Size)
+
+// This macro is for use by MC-generated code and should not be used directly.
+#define _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)
+
+//
+// Enablement check macro for event "ETW_MI_FREE"
+//
+#define EventEnabledETW_MI_FREE() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)
+#define EventEnabledETW_MI_FREE_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)
+
+//
+// Event write macros for event "ETW_MI_FREE"
+//
+#define EventWriteETW_MI_FREE(Address, Size) \
+        MCGEN_EVENT_ENABLED(ETW_MI_FREE) \
+        ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) : 0
+#define EventWriteETW_MI_FREE_AssumeEnabled(Address, Size) \
+        _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size)
+#define EventWriteETW_MI_FREE_ForContext(pContext, Address, Size) \
+        MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_FREE) \
+        ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&(pContext)->Context, &ETW_MI_FREE, Address, Size) : 0
+#define EventWriteETW_MI_FREE_ForContextAssumeEnabled(pContext, Address, Size) \
+        _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_FREE, Address, Size)
+
+// This macro is for use by MC-generated code and should not be used directly.
+#define _mcgen_TEMPLATE_FOR_ETW_MI_FREE _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)
+
+#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION
+
+//
+// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:
+// Define this macro to have the compiler skip the generated functions in this
+// header.
+//
+#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION
+
+//
+// Template Functions
+//
+
+//
+// Function for template "ETW_CUSTOM_HEAP_ALLOC_DATA" (and possibly others).
+// This function is for use by MC-generated code and should not be used directly.
+//
+#ifndef McTemplateU0xx_def
+#define McTemplateU0xx_def
+ETW_INLINE
+ULONG
+_mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)(
+    _In_ PMCGEN_TRACE_CONTEXT Context,
+    _In_ PCEVENT_DESCRIPTOR Descriptor,
+    _In_ const unsigned __int64  _Arg0,
+    _In_ const unsigned __int64  _Arg1
+    )
+{
+#define McTemplateU0xx_ARGCOUNT 2
+
+    EVENT_DATA_DESCRIPTOR EventData[McTemplateU0xx_ARGCOUNT + 1];
+
+    EventDataDescCreate(&EventData[1],&_Arg0, sizeof(const unsigned __int64)  );
+
+    EventDataDescCreate(&EventData[2],&_Arg1, sizeof(const unsigned __int64)  );
+
+    return McGenEventWrite(Context, Descriptor, NULL, McTemplateU0xx_ARGCOUNT + 1, EventData);
+}
+#endif // McTemplateU0xx_def
+
+#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION
+
+#if defined(__cplusplus)
+}
+#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.man b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.man
new file mode 100644
index 0000000000000000000000000000000000000000..cfd1f8a9eaacd50af63f1e28f9540aa88c20f90c
GIT binary patch
literal 3926
zcmeH~T~8B16o${WiT`2c+NGc<*i;EYh$d8xl<0*CS-Mb~v<qPu>PP(R>hsQY*nU!q
z$b})3?##}d?{nTW+uy%xwr*doYaNU1!Vc}sa%<a(W%kk*Y}poVi8<mVvnKl~r<t|f
z>7F%g+hVAmL$hwL?4dodnmuAKhUXm0)X9|WHj>XRahh@~SWDIkbduX;B#u6^Q>@U=
zDO8U+KXa0*th&%f*z^Udh4ol@uE=Q&`emUsh_CA`FOXgI{i-`XZ9C#bR1yBm=PJ*p
z9kVN$J6O;h;8HY>p)RnhY8A#Hb?z)_!y(Iaen(I)@-9CrSSp(;_Jn9I*$S&ATjP1?
zVxB>pV@LVsy;^jZr7r$HNAm06TcUiI`l@~F$Mt$E%Shfd3O+h1vFhR9a8yQZ@wpne
zr3bI-p=VEdo{)#uWxSVJeYQF|-5tnq>~f+CP~A0&{v=(up=ngEDl>5!$EDwPRaNjW
zXj|wbG$OwmwaW-hMvBK%pbm3wpic7<iGe1}wLX@G{?G|Bd>1O^dzbxT%*13+SP9h-
zI~rA5hapTVnk|qmiIVYy{__+x9f7OV4j3`g4;{{8_SWnLBSu2PUc%~`t%Ae^>J`SS
zdtZg-r<0xAH*_ALtK;Nv(d9nbKK1jK=Ld)I(jQrKhBjgToR#Wm8{0a}@6ZuEO<gbQ
zB3m5V%^3-v%%-25hY;g1&y$zH6XrqQ2)nL|zy>(lvG=y=Jh{M!4!-$(E)!vYUrf47
z<hkOaGap-@>nf4W$e&QFOovV_$>J%X*KN>oXI@jt%BJms>IU}I$<7<wvJG;y&ofxL
zZ?Ac4@EfiF;Qh3@Hq?u*mxUl}o``PSN9^9363^xS{`lRzaEI{>Hr{PChchs%+mx{%
zt(fa_PM4r63?1h#YOk~;byc5`>%q>sLHA1gohNq{qOREhxu<<B25jv)bQOG~tvEXM
zsbYvycWRYgeO<KaNEs4R@T0H57CiG+erb6HDME8{V+nzO*!Nwm{Em&P|NqBp)%s}N
z&KLRG82>y~`}YVhGe0?R_ceQ8vt^BpSNp6kErj_0hUNFyWQ1IOZ|GEgWBPwYFLgFu
Oo!*uqEBu%Ae18C_t1cS=

literal 0
HcmV?d00001

diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/prim.c
new file mode 100644
index 00000000..51ad3104
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/prim.c
@@ -0,0 +1,1124 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// This file is included in `src/prim/prim.c`
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"
+#include <stdio.h>   // fputs, stderr
+#include <stdlib.h>  // atexit
+
+// xbox has no console IO
+#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM)
+#define MI_HAS_CONSOLE_IO
+#endif
+
+//---------------------------------------------
+// Dynamically bind Windows API points for portability
+//---------------------------------------------
+
+#if defined(_MSC_VER)
+#pragma warning(disable:4996)   // don't use GetVersionExW
+#endif
+
+static DWORD win_major_version = 6;
+static DWORD win_minor_version = 0;
+
+// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
+// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
+// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB)
+// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's.
+typedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E {
+  MiMemExtendedParameterInvalidType = 0,
+  MiMemExtendedParameterAddressRequirements,
+  MiMemExtendedParameterNumaNode,
+  MiMemExtendedParameterPartitionHandle,
+  MiMemExtendedParameterUserPhysicalHandle,
+  MiMemExtendedParameterAttributeFlags,
+  MiMemExtendedParameterMax
+} MI_MEM_EXTENDED_PARAMETER_TYPE;
+
+typedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S {
+  struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type;
+  union  { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg;
+} MI_MEM_EXTENDED_PARAMETER;
+
+typedef struct MI_MEM_ADDRESS_REQUIREMENTS_S {
+  PVOID  LowestStartingAddress;
+  PVOID  HighestEndingAddress;
+  SIZE_T Alignment;
+} MI_MEM_ADDRESS_REQUIREMENTS;
+
+#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE   0x00000010
+
+#include <winternl.h>
+typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);
+typedef LONG  (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);  // avoid NTSTATUS as it is not defined on xbox (pr #1084)
+static PVirtualAlloc2 pVirtualAlloc2 = NULL;
+static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;
+
+// Similarly, GetNumaProcessorNodeEx is only supported since Windows 7  (and GetNumaNodeProcessorMask is not supported on xbox)
+typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER;
+
+typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber);
+typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber);
+typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask);
+typedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber);
+typedef BOOL (__stdcall* PGetNumaNodeProcessorMask)(UCHAR Node, PULONGLONG ProcessorMask);
+typedef BOOL (__stdcall* PGetNumaHighestNodeNumber)(PULONG Node);
+static PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL;
+static PGetNumaProcessorNodeEx      pGetNumaProcessorNodeEx = NULL;
+static PGetNumaNodeProcessorMaskEx  pGetNumaNodeProcessorMaskEx = NULL;
+static PGetNumaProcessorNode        pGetNumaProcessorNode = NULL;
+static PGetNumaNodeProcessorMask    pGetNumaNodeProcessorMask = NULL;
+static PGetNumaHighestNodeNumber    pGetNumaHighestNodeNumber = NULL;
+
+// Not available on xbox
+typedef SIZE_T(__stdcall* PGetLargePageMinimum)(VOID);
+static PGetLargePageMinimum pGetLargePageMinimum = NULL;
+
+// Available after Windows XP
+typedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalMemoryInKilobytes );
+typedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation);
+
+
+
+//---------------------------------------------
+// Enable large page support dynamically (if possible)
+//---------------------------------------------
+
+static bool win_enable_large_os_pages_once(size_t* large_page_size)
+{
+  if (pGetLargePageMinimum==NULL) return false;  // no large page support (xbox etc.)
+
+  // Try to see if large OS pages are supported
+  // To use large pages on Windows, we first need access permission
+  // Set "Lock pages in memory" permission in the group policy editor
+  // <https://devblogs.microsoft.com/oldnewthing/20110128-00/?p=11643>
+  unsigned long err = 0;
+  HANDLE token = NULL;
+  BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
+  if (ok) {
+    TOKEN_PRIVILEGES tp;
+    ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid);
+    if (ok) {
+      tp.PrivilegeCount = 1;
+      tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+      ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);
+      if (ok) {
+        err = GetLastError();
+        ok = (err == ERROR_SUCCESS);
+        if (ok && large_page_size != NULL && pGetLargePageMinimum != NULL) {
+          *large_page_size = (*pGetLargePageMinimum)();
+        }
+      }
+    }
+    CloseHandle(token);
+  }
+  if (!ok) {
+    if (err == 0) err = GetLastError();
+    _mi_warning_message("cannot enable large OS page support, error %lu\n", err);
+  }
+  return (ok!=0);
+}
+
+static bool win_enable_large_os_pages(size_t* large_page_size) {
+  mi_atomic_do_once {
+    win_enable_large_os_pages_once(large_page_size);
+  }
+  return (_mi_os_large_page_size() > 0);
+}
+
+
+//---------------------------------------------
+// Initialize
+//---------------------------------------------
+
+static DWORD win_allocation_granularity = 64*MI_KiB;
+
+void _mi_prim_mem_init( mi_os_mem_config_t* config )
+{
+  config->has_overcommit = false;
+  config->has_partial_free = false;
+  config->has_virtual_reserve = true;
+
+  // get the page size
+  SYSTEM_INFO si; _mi_memzero_var(si);
+  GetSystemInfo(&si);
+  if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; }
+  if (si.dwAllocationGranularity > 0) {
+    config->alloc_granularity = si.dwAllocationGranularity;
+    win_allocation_granularity = si.dwAllocationGranularity;
+  }
+  // get virtual address bits
+  if ((uintptr_t)si.lpMaximumApplicationAddress > 0) {
+    const size_t vbits = MI_SIZE_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);
+    config->virtual_address_bits = vbits;
+  }
+
+  // get the VirtualAlloc2 function
+  HINSTANCE hDll = LoadLibrary(TEXT("kernelbase.dll"));
+  if (hDll != NULL) {
+    // use VirtualAlloc2FromApp if possible as it is available to Windows store apps
+    pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp");
+    if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2");
+    FreeLibrary(hDll);
+  }
+  // NtAllocateVirtualMemoryEx is used for huge page allocation
+  hDll = LoadLibrary(TEXT("ntdll.dll"));
+  if (hDll != NULL) {
+    pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx");
+    FreeLibrary(hDll);
+  }
+  // Try to use Win7+ numa API
+  hDll = LoadLibrary(TEXT("kernel32.dll"));
+  if (hDll != NULL) {
+    pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx");
+    pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx");
+    pGetNumaNodeProcessorMaskEx = (PGetNumaNodeProcessorMaskEx)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMaskEx");
+    pGetNumaProcessorNode = (PGetNumaProcessorNode)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNode");
+    pGetNumaNodeProcessorMask = (PGetNumaNodeProcessorMask)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMask");
+    pGetNumaHighestNodeNumber = (PGetNumaHighestNodeNumber)(void (*)(void))GetProcAddress(hDll, "GetNumaHighestNodeNumber");
+    pGetLargePageMinimum = (PGetLargePageMinimum)(void (*)(void))GetProcAddress(hDll, "GetLargePageMinimum");
+    // Get physical memory (not available on XP, so check dynamically)
+    PGetPhysicallyInstalledSystemMemory pGetPhysicallyInstalledSystemMemory = (PGetPhysicallyInstalledSystemMemory)(void (*)(void))GetProcAddress(hDll,"GetPhysicallyInstalledSystemMemory");
+    if (pGetPhysicallyInstalledSystemMemory != NULL) {
+      ULONGLONG memInKiB = 0;
+      if ((*pGetPhysicallyInstalledSystemMemory)(&memInKiB)) {
+        if (memInKiB > 0 && memInKiB <= SIZE_MAX) {
+          config->physical_memory_in_kib = (size_t)memInKiB;
+        }
+      }
+    }
+    // Get Windows version
+    PGetVersionExW pGetVersionExW = (PGetVersionExW)(void (*)(void))GetProcAddress(hDll, "GetVersionExW");
+    if (pGetVersionExW != NULL) {
+      OSVERSIONINFOW version; _mi_memzero_var(version);
+      version.dwOSVersionInfoSize = sizeof(version);
+      if ((*pGetVersionExW)(&version)) {
+        win_major_version = version.dwMajorVersion;
+        win_minor_version = version.dwMinorVersion;
+      }
+    }
+    FreeLibrary(hDll);
+  }
+  // Enable large/huge OS page support?
+  if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
+    win_enable_large_os_pages(&config->large_page_size);
+  }
+}
+
+
+//---------------------------------------------
+// Free
+//---------------------------------------------
+
+int _mi_prim_free(void* addr, size_t size ) {
+  MI_UNUSED(size);
+  DWORD errcode = 0;
+  bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);
+  if (err) { errcode = GetLastError(); }
+  if (errcode == ERROR_INVALID_ADDRESS) {
+    // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside
+    // the memory region returned by VirtualAlloc; in that case we need to free using
+    // the start of the region.
+    MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info);
+    VirtualQuery(addr, &info, sizeof(info));
+    if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) {
+      errcode = 0;
+      err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0);
+      if (err) { errcode = GetLastError(); }
+    }
+  }
+  return (int)errcode;
+}
+
+
+//---------------------------------------------
+// VirtualAlloc
+//---------------------------------------------
+
+static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) {
+  #if (MI_INTPTR_SIZE >= 8)
+  // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations
+  if (addr == NULL) {
+    void* hint = _mi_os_get_aligned_hint(try_alignment,size);
+    if (hint != NULL) {
+      void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE);
+      if (p != NULL) return p;
+      _mi_verbose_message("warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), hint, try_alignment, flags);
+      // fall through on error
+    }
+  }
+  #endif
+  // on modern Windows try use VirtualAlloc2 for aligned allocation
+  if (addr == NULL && try_alignment > win_allocation_granularity && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
+    MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };
+    reqs.Alignment = try_alignment;
+    MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };
+    param.Type.Type = MiMemExtendedParameterAddressRequirements;
+    param.Arg.Pointer = &reqs;
+    void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, &param, 1);
+    if (p != NULL) return p;
+    _mi_warning_message("unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags);
+    // fall through on error
+  }
+  // last resort
+  return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
+}
+
+static bool win_is_out_of_memory_error(DWORD err) {
+  switch (err) {
+    case ERROR_COMMITMENT_MINIMUM:
+    case ERROR_COMMITMENT_LIMIT:
+    case ERROR_PAGEFILE_QUOTA:
+    case ERROR_NOT_ENOUGH_MEMORY:
+      return true;
+    default:
+      return false;
+  }
+}
+
+static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {
+  long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000);  // at most 2 seconds
+  if (max_retry_msecs == 1) { max_retry_msecs = 100; }  // if one sets the option to "true"
+  for (long tries = 1; tries <= 10; tries++) {          // try at most 10 times (=2200ms)
+    void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags);
+    if (p != NULL) {
+      // success, return the address
+      return p;
+    }
+    else if (max_retry_msecs > 0 && (try_alignment <= 8*MI_MiB) &&
+              (flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 &&
+              win_is_out_of_memory_error(GetLastError())) {
+      // if committing regular memory and being out-of-memory,
+      // keep trying for a bit in case memory frees up after all. See issue #894
+      _mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags);
+      long sleep_msecs = tries*40;  // increasing waits
+      if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; }
+      max_retry_msecs -= sleep_msecs;
+      Sleep(sleep_msecs);
+    }
+    else {
+      // otherwise return with an error
+      break;
+    }
+  }
+  return NULL;
+}
+
+static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {
+  mi_assert_internal(!(large_only && !allow_large));
+  static _Atomic(size_t) large_page_try_ok; // = 0;
+  void* p = NULL;
+  // Try to allocate large OS pages (2MiB) if allowed or required.
+  if ((large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))
+      && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0)
+  {
+    size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
+    if (!large_only && try_ok > 0) {
+      // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.
+      // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times.
+      mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);
+    }
+    else {
+      // large OS pages must always reserve and commit.
+      *is_large = true;
+      p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES);
+      if (large_only) return p;
+      // fall back to non-large page allocation on error (`p == NULL`).
+      if (p == NULL) {
+        mi_atomic_store_release(&large_page_try_ok,10UL);  // on error, don't try again for the next N allocations
+      }
+    }
+  }
+  // Fall back to regular page allocation
+  if (p == NULL) {
+    *is_large = ((flags&MEM_LARGE_PAGES) != 0);
+    p = win_virtual_alloc_prim(addr, size, try_alignment, flags);
+  }
+  //if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); }
+  return p;
+}
+
+int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
+  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
+  mi_assert_internal(commit || !allow_large);
+  mi_assert_internal(try_alignment > 0);
+  *is_zero = true;
+  int flags = MEM_RESERVE;
+  if (commit) { flags |= MEM_COMMIT; }
+  *addr = win_virtual_alloc(hint_addr, size, try_alignment, flags, false, allow_large, is_large);
+  return (*addr != NULL ? 0 : (int)GetLastError());
+}
+
+
+//---------------------------------------------
+// Commit/Reset/Protect
+//---------------------------------------------
+#ifdef _MSC_VER
+#pragma warning(disable:6250)   // suppress warning calling VirtualFree without MEM_RELEASE (for decommit)
+#endif
+
+int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
+  *is_zero = false;
+  /*
+  // zero'ing only happens on an initial commit... but checking upfront seems expensive..
+  _MEMORY_BASIC_INFORMATION meminfo; _mi_memzero_var(meminfo);
+  if (VirtualQuery(addr, &meminfo, size) > 0) {
+    if ((meminfo.State & MEM_COMMIT) == 0) {
+      *is_zero = true;
+    }
+  }
+  */
+  // commit
+  void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
+  if (p == NULL) return (int)GetLastError();
+  return 0;
+}
+
+int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {
+  BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT);
+  *needs_recommit = true;  // for safety, assume always decommitted even in the case of an error.
+  return (ok ? 0 : (int)GetLastError());
+}
+
+int _mi_prim_reset(void* addr, size_t size) {
+  void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
+  mi_assert_internal(p == addr);
+  #if 0
+  if (p != NULL) {
+    VirtualUnlock(addr,size); // VirtualUnlock after MEM_RESET removes the memory directly from the working set
+  }
+  #endif
+  return (p != NULL ? 0 : (int)GetLastError());
+}
+
+int _mi_prim_reuse(void* addr, size_t size) {
+  MI_UNUSED(addr); MI_UNUSED(size);
+  return 0;
+}
+
+int _mi_prim_protect(void* addr, size_t size, bool protect) {
+  DWORD oldprotect = 0;
+  BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect);
+  return (ok ? 0 : (int)GetLastError());
+}
+
+
+//---------------------------------------------
+// Huge page allocation
+//---------------------------------------------
+
+static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node)
+{
+  const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE;
+
+  win_enable_large_os_pages(NULL);
+
+  MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} };
+  // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
+  static _Atomic(size_t) mi_huge_pages_available = MI_ATOMIC_VAR_INIT(1);
+  if (pNtAllocateVirtualMemoryEx != NULL && mi_atomic_load_acquire(&mi_huge_pages_available) != 0) {
+    params[0].Type.Type = MiMemExtendedParameterAttributeFlags;
+    params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
+    ULONG param_count = 1;
+    if (numa_node >= 0) {
+      param_count++;
+      params[1].Type.Type = MiMemExtendedParameterNumaNode;
+      params[1].Arg.ULong = (unsigned)numa_node;
+    }
+    SIZE_T psize = size;
+    void* base = hint_addr;
+    LONG err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count);
+    if (err == 0 && base != NULL) {
+      return base;
+    }
+    else {
+      // fall back to regular large pages
+      mi_atomic_store_release(&mi_huge_pages_available,0); // don't try further huge pages
+      _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err);
+    }
+  }
+  // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation
+  if (pVirtualAlloc2 != NULL && numa_node >= 0) {
+    params[0].Type.Type = MiMemExtendedParameterNumaNode;
+    params[0].Arg.ULong = (unsigned)numa_node;
+    return (*pVirtualAlloc2)(GetCurrentProcess(), hint_addr, size, flags, PAGE_READWRITE, params, 1);
+  }
+
+  // otherwise use regular virtual alloc on older windows
+  return VirtualAlloc(hint_addr, size, flags, PAGE_READWRITE);
+}
+
+int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
+  *is_zero = true;
+  *addr = _mi_prim_alloc_huge_os_pagesx(hint_addr,size,numa_node);
+  return (*addr != NULL ? 0 : (int)GetLastError());
+}
+
+
+//---------------------------------------------
+// Numa nodes
+//---------------------------------------------
+
+size_t _mi_prim_numa_node(void) {
+  USHORT numa_node = 0;
+  if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) {
+    // Extended API is supported
+    MI_PROCESSOR_NUMBER pnum;
+    (*pGetCurrentProcessorNumberEx)(&pnum);
+    USHORT nnode = 0;
+    BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode);
+    if (ok) { numa_node = nnode; }
+  }
+  else if (pGetNumaProcessorNode != NULL) {
+    // Vista or earlier, use older API that is limited to 64 processors. Issue #277
+    DWORD pnum = GetCurrentProcessorNumber();
+    UCHAR nnode = 0;
+    BOOL ok = pGetNumaProcessorNode((UCHAR)pnum, &nnode);
+    if (ok) { numa_node = nnode; }
+  }
+  return numa_node;
+}
+
+size_t _mi_prim_numa_node_count(void) {
+  ULONG numa_max = 0;
+  if (pGetNumaHighestNodeNumber!=NULL) {
+    (*pGetNumaHighestNodeNumber)(&numa_max);
+  }
+  // find the highest node number that has actual processors assigned to it. Issue #282
+  while (numa_max > 0) {
+    if (pGetNumaNodeProcessorMaskEx != NULL) {
+      // Extended API is supported
+      GROUP_AFFINITY affinity;
+      if ((*pGetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) {
+        if (affinity.Mask != 0) break;  // found the maximum non-empty node
+      }
+    }
+    else {
+      // Vista or earlier, use older API that is limited to 64 processors.
+      ULONGLONG mask;
+      if (pGetNumaNodeProcessorMask != NULL) {
+        if ((*pGetNumaNodeProcessorMask)((UCHAR)numa_max, &mask)) {
+          if (mask != 0) break; // found the maximum non-empty node
+        }
+      };
+    }
+    // max node was invalid or had no processor assigned, try again
+    numa_max--;
+  }
+  return ((size_t)numa_max + 1);
+}
+
+
+//----------------------------------------------------------------
+// Clock
+//----------------------------------------------------------------
+
+static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {
+  static LARGE_INTEGER mfreq; // = 0
+  if (mfreq.QuadPart == 0LL) {
+    LARGE_INTEGER f;
+    QueryPerformanceFrequency(&f);
+    mfreq.QuadPart = f.QuadPart/1000LL;
+    if (mfreq.QuadPart == 0) mfreq.QuadPart = 1;
+  }
+  return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart);
+}
+
+mi_msecs_t _mi_prim_clock_now(void) {
+  LARGE_INTEGER t;
+  QueryPerformanceCounter(&t);
+  return mi_to_msecs(t);
+}
+
+
+//----------------------------------------------------------------
+// Process Info
+//----------------------------------------------------------------
+
+#include <psapi.h>
+
+static mi_msecs_t filetime_msecs(const FILETIME* ftime) {
+  ULARGE_INTEGER i;
+  i.LowPart = ftime->dwLowDateTime;
+  i.HighPart = ftime->dwHighDateTime;
+  mi_msecs_t msecs = (i.QuadPart / 10000); // FILETIME is in 100 nano seconds
+  return msecs;
+}
+
+typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD);
+static PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL;
+
+void _mi_prim_process_info(mi_process_info_t* pinfo)
+{
+  FILETIME ct;
+  FILETIME ut;
+  FILETIME st;
+  FILETIME et;
+  GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut);
+  pinfo->utime = filetime_msecs(&ut);
+  pinfo->stime = filetime_msecs(&st);
+
+  // load psapi on demand
+  mi_atomic_do_once {
+    HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll"));
+    if (hDll != NULL) {
+      pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo");
+      // FreeLibrary(hDll);  // don't free
+    }
+  }
+
+  // get process info
+  PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);
+  if (pGetProcessMemoryInfo != NULL) {
+    pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));
+  }
+  pinfo->current_rss    = (size_t)info.WorkingSetSize;
+  pinfo->peak_rss       = (size_t)info.PeakWorkingSetSize;
+  pinfo->current_commit = (size_t)info.PagefileUsage;
+  pinfo->peak_commit    = (size_t)info.PeakPagefileUsage;
+  pinfo->page_faults    = (size_t)info.PageFaultCount;
+}
+
+//----------------------------------------------------------------
+// Output
+//----------------------------------------------------------------
+
+void _mi_prim_out_stderr( const char* msg )
+{
+  // on windows with redirection, the C runtime cannot handle locale dependent output
+  // after the main thread closes so we use direct console output.
+  if (!_mi_preloading()) {
+    // _cputs(msg);  // _cputs cannot be used as it aborts when failing to lock the console
+    static HANDLE hcon = INVALID_HANDLE_VALUE;
+    static bool hconIsConsole = false;
+    if (hcon == INVALID_HANDLE_VALUE) {
+      hcon = GetStdHandle(STD_ERROR_HANDLE);
+      #ifdef MI_HAS_CONSOLE_IO
+      CONSOLE_SCREEN_BUFFER_INFO sbi;
+      hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi));
+      #endif
+    }
+    const size_t len = _mi_strlen(msg);
+    if (len > 0 && len < UINT32_MAX) {
+      DWORD written = 0;
+      if (hconIsConsole) {
+        #ifdef MI_HAS_CONSOLE_IO
+        WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);
+        #endif
+      }
+      else if (hcon != INVALID_HANDLE_VALUE) {
+        // use direct write if stderr was redirected
+        WriteFile(hcon, msg, (DWORD)len, &written, NULL);
+      }
+      else {
+        // finally fall back to fputs after all
+        fputs(msg, stderr);
+      }
+    }
+  }
+}
+
+
+//----------------------------------------------------------------
+// Environment
+//----------------------------------------------------------------
+
+// On Windows use GetEnvironmentVariable instead of getenv to work
+// reliably even when this is invoked before the C runtime is initialized.
+// i.e. when `_mi_preloading() == true`.
+// Note: on windows, environment names are not case sensitive.
+bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+  result[0] = 0;
+  const size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size);
+  return (len > 0 && len < result_size);
+}
+
+
+//----------------------------------------------------------------
+// Random
+//----------------------------------------------------------------
+
+#if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus)
+// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using
+// dynamic overriding, we observed it can raise an exception when compiled with C++, and
+// sometimes deadlocks when also running under the VS debugger.
+// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom.
+// To be continued..
+#pragma comment (lib,"advapi32.lib")
+#define RtlGenRandom  SystemFunction036
+mi_decl_externc BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength);
+
+bool _mi_prim_random_buf(void* buf, size_t buf_len) {
+  return (RtlGenRandom(buf, (ULONG)buf_len) != 0);
+}
+
+#else
+
+#ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG
+#define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002
+#endif
+
+typedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG);
+static  PBCryptGenRandom pBCryptGenRandom = NULL;
+
+bool _mi_prim_random_buf(void* buf, size_t buf_len) {
+  mi_assert(buf_len <= ULONG_MAX);
+  if (buf_len > ULONG_MAX) return false;
+  mi_atomic_do_once {
+    HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll"));
+    if (hDll != NULL) {
+      pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom");
+      // FreeLibrary(hDll);  // don't free
+    }
+  }
+  if (pBCryptGenRandom == NULL) return false;
+  return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0);
+}
+
+#endif  // MI_USE_RTLGENRANDOM
+
+
+//----------------------------------------------------------------
+// Thread pool?
+//----------------------------------------------------------------
+
+bool _mi_prim_thread_is_in_threadpool(void) {
+#if (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64)
+  if (win_major_version >= 6) {
+    // check if this thread belongs to a windows threadpool
+    // see: <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm>
+    struct _TEB* const teb = NtCurrentTeb();
+    void* const pool_data = *((void**)((uint8_t*)teb + (MI_SIZE_BITS == 32 ? 0x0F90 : 0x1778)));
+    return (pool_data != NULL);
+  }
+#endif
+  return false;
+}
+
+void _mi_prim_thread_yield(void) {
+  SwitchToThread();
+}
+
+//----------------------------------------------------------------
+// Process & Thread Init/Done
+//----------------------------------------------------------------
+
+//static void mi_debug_out(const char* s) {
+//  HANDLE h = GetStdHandle(STD_ERROR_HANDLE);
+//  WriteConsole(h, s, (DWORD)_mi_strlen(s), NULL, NULL);
+//}
+
+static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
+  MI_UNUSED(reserved);
+  MI_UNUSED(module);
+  if (reason==DLL_PROCESS_ATTACH) {
+    _mi_auto_process_init();
+  }
+  else if (reason==DLL_PROCESS_DETACH) {
+    _mi_auto_process_done();
+  }
+  else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) {
+    _mi_thread_done(NULL);
+  }
+}
+
+
+/* -----------------------------------------------------------------------
+   Auto initialize and finalize mimalloc on process and thread start/end.
+   By default we use a combination of CRT init and TLS sections for
+   both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`).
+------------------------------------------------------------------------- */
+#if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS)
+  #if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER)
+    #define MI_WIN_INIT_USE_CRT_TLS      1  
+  #else
+    #define MI_WIN_INIT_USE_TLS_DLLMAIN  1  /* default for Intel ICX, see issue #1268 */  
+  #endif
+#endif
+
+#if defined(MI_WIN_INIT_USE_CRT_TLS)
+  #define MI_PRIM_HAS_PROCESS_ATTACH  1
+  // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
+  void _mi_prim_thread_init_auto_done(void) {}
+  void _mi_prim_thread_done_auto_done(void) {}
+  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+    MI_UNUSED(theap);
+  }
+
+  static bool mi_module_is_dll(PVOID mod) {
+    if (mod==NULL) return false;
+    PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;
+    PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);
+    return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);
+  }
+
+  static bool mi_current_module_is_dll(void) {
+    HMODULE mod = NULL;
+    const BOOL ok = GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCSTR)mi_current_module_is_dll, &mod);
+    return (ok && mi_module_is_dll(mod));
+  }
+
+  // Hook into CRT initialization and finalization.
+  static void mi_cdecl mi_crt_done(void) {
+    // mi_debug_out(mi_current_module_is_dll() ? "crt dll done\n" : "crt exe done\n");
+    mi_win_main(NULL, DLL_PROCESS_DETACH, 0);
+  }
+
+  static int mi_cdecl mi_crt_init(void) {
+    // mi_debug_out(mi_current_module_is_dll() ? "crt dll init\n" : "crt exe init\n");
+    if (mi_current_module_is_dll()) {
+      // in a dll, atexit (crt_done) is called after tls process detach
+      atexit(&mi_crt_done);
+    }
+    return 0;
+  }
+
+  // We also hook into the Windows loader TLS initialization and finalization.
+  static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {
+    if (reason == DLL_THREAD_ATTACH) {
+      //mi_debug_out("tls thread attach\n");
+      mi_win_main(module, reason, reserved);
+    }
+    else if (reason == DLL_PROCESS_ATTACH) {
+      // mi_debug_out(mi_module_is_dll(module) ? "tls dll process attach\n" : "tls exe process attach\n");
+      // tls process attach is always called before crt init
+      mi_win_main(module, reason, reserved);
+    }
+  }
+
+  static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {
+    if (reason == DLL_THREAD_DETACH) {
+      //mi_debug_out("tls thread detach\n");
+      mi_win_main(module, reason, reserved);
+    }
+    else if (reason == DLL_PROCESS_DETACH) {
+      // mi_debug_out(mi_module_is_dll(module) ? "tls dll process detach\n" : "tls exe process detach\n");
+      if (!mi_module_is_dll(module)) {
+        // in an exe, tls process detach is called after atexit (mi_crt_done)
+        mi_win_main(module, reason, reserved);
+      }
+    }
+  }
+
+  // Set up TLS callbacks in a statically linked library by using special data sections.
+  // See <https://stackoverflow.com/questions/14538159/tls-callback-in-windows>
+  // We use 2 entries to ensure we call attach events before constructors
+  // are called, and detach events after destructors are called.
+  // We also set up a CRT initialization callback.
+  #if defined(__cplusplus)
+  extern "C" {
+  #endif
+
+  typedef int (mi_cdecl* mi_crt_callback_t)(void);
+  #if defined(_WIN64)
+    #pragma comment(linker, "/INCLUDE:_tls_used")
+    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
+    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
+    #pragma const_seg(".CRT$XLB")
+      extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
+      const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
+    #pragma const_seg()
+    #pragma const_seg(".CRT$XLY")
+      extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
+      const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
+    #pragma const_seg()
+    #pragma const_seg(".CRT$XIB")
+      extern const mi_crt_callback_t _mi_crt_callback_init[];
+      const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
+    #pragma const_seg()
+  #else
+    #pragma comment(linker, "/INCLUDE:__tls_used")
+    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
+    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
+    #pragma data_seg(".CRT$XLB")
+      PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
+    #pragma data_seg()
+    #pragma data_seg(".CRT$XLY")
+      PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
+    #pragma data_seg()
+    #pragma data_seg(".CRT$XIB")
+      mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
+    #pragma data_seg()
+  #endif
+
+  #if defined(__cplusplus)
+  }
+  #endif
+
+/* ----------------------------------------------------------------
+  Here we use a combination of the raw DLL main and TLS sections for
+  both static and dynamic linkage. This is perhaps the most robust as
+  the raw DLL main ensure we can close mimalloc latest possible.
+  However, it can lead to link errors with on `_pRawDllMain` if other
+  libraries also try to hook into this (like Boost).
+*/
+#elif defined(MI_WIN_INIT_USE_RAW_DLLMAIN)
+  #define MI_PRIM_HAS_PROCESS_ATTACH  1
+  // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
+  void _mi_prim_thread_init_auto_done(void) {}
+  void _mi_prim_thread_done_auto_done(void) {}
+  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+    MI_UNUSED(theap);
+  }
+
+  // If linked into a DLL module, this raw entry is called before the CRT attach and
+  // after the CRT detach through the CRT _pRawDllMain pointer.
+  static BOOL NTAPI mi_dll_main_raw(PVOID module, DWORD reason, LPVOID reserved) {
+    mi_win_main(module, reason, reserved);
+    return TRUE;
+  }
+
+  // Set the value of the CRT _pRawDllMain pointer
+  #if defined(__cplusplus)
+  extern "C"
+  #endif
+  PVOID _pRawDllMain = (PVOID)&mi_dll_main_raw;
+
+  // We also hook into the Windows loader TLS initialization and finalization.
+  // If we are linked into an EXE module we rely on these as `mi_dll_main_raw`
+  // is not called (and otherwise we ignore the TLS callbacks by checking if we are in a DLL).
+  static bool mi_module_is_dll(PVOID mod) {
+    if (mod==NULL) return false;
+    PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;
+    PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);
+    return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);
+  }
+
+  static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {
+    if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {
+      if (!mi_module_is_dll(module)) {
+        mi_win_main(module, reason, reserved);
+      }
+    }
+  }
+
+  static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {
+    if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {
+      if (!mi_module_is_dll(module)) {
+        mi_win_main(module, reason, reserved);
+      }
+    }
+  }
+
+  // Set up TLS callbacks in a statically linked library by using special data sections.
+  // See <https://stackoverflow.com/questions/14538159/tls-callback-in-windows>
+  // We use 2 entries to ensure we call attach events before constructors
+  // are called, and detach events after destructors are called.
+  #if defined(__cplusplus)
+  extern "C" {
+  #endif
+
+  #if defined(_WIN64)
+    #pragma comment(linker, "/INCLUDE:_tls_used")
+    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
+    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
+    #pragma const_seg(".CRT$XLB")
+      extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
+      const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
+    #pragma const_seg()
+    #pragma const_seg(".CRT$XLY")
+      extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
+      const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
+    #pragma const_seg()
+  #else
+    #pragma comment(linker, "/INCLUDE:__tls_used")
+    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
+    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
+    #pragma data_seg(".CRT$XLB")
+      PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
+    #pragma data_seg()
+    #pragma data_seg(".CRT$XLY")
+      PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
+    #pragma data_seg()
+  #endif
+
+  #if defined(__cplusplus)
+  }
+  #endif
+
+/* --------------------------------------------------------------------
+   Legacy options: DllMain, TLS, and FLS
+   --------------------------------------------------------------------*/
+#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && defined(MI_SHARED_LIB)
+  #define MI_PRIM_HAS_PROCESS_ATTACH  1
+
+  // Windows DLL: easy to hook into process_init and thread_done
+  BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {
+    mi_win_main((PVOID)inst,reason,reserved);
+    return TRUE;
+  }
+
+  // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
+  void _mi_prim_thread_init_auto_done(void) { }
+  void _mi_prim_thread_done_auto_done(void) { }
+  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+    MI_UNUSED(theap);
+  }
+
+#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN)
+  #define MI_PRIM_HAS_PROCESS_ATTACH  1
+
+  static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) {
+    if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {
+      mi_win_main(module, reason, reserved);
+    }
+  }
+  static void NTAPI mi_win_main_detach(PVOID module, DWORD reason, LPVOID reserved) {
+    if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {
+      mi_win_main(module, reason, reserved);
+    }
+  }
+
+  // Set up TLS callbacks in a statically linked library by using special data sections.
+  // See <https://stackoverflow.com/questions/14538159/tls-callback-in-windows>
+  // We use 2 entries to ensure we call attach events before constructors
+  // are called, and detach events after destructors are called.
+  #if defined(__cplusplus)
+  extern "C" {
+  #endif
+
+  #if defined(_WIN64)
+    #pragma comment(linker, "/INCLUDE:_tls_used")
+    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
+    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
+    #pragma const_seg(".CRT$XLB")
+    extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
+    const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };
+    #pragma const_seg()
+    #pragma const_seg(".CRT$XLY")
+    extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
+    const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
+    #pragma const_seg()
+  #else
+    #pragma comment(linker, "/INCLUDE:__tls_used")
+    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
+    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
+    #pragma data_seg(".CRT$XLB")
+    PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };
+    #pragma data_seg()
+    #pragma data_seg(".CRT$XIY")
+    PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
+    #pragma data_seg()
+  #endif
+
+  #if defined(__cplusplus)
+  }
+  #endif
+
+  // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
+  void _mi_prim_thread_init_auto_done(void) { }
+  void _mi_prim_thread_done_auto_done(void) { }
+  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+    MI_UNUSED(theap);
+  }
+
+#elif defined(MI_WIN_INIT_USE_FLS) // deprecated: statically linked, use fiber api
+
+  #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`)
+    // MSVC: use data section magic for static libraries
+    // See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
+    #define MI_PRIM_HAS_PROCESS_ATTACH 1
+
+    static int mi_process_attach(void) {
+      mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL);
+      atexit(&_mi_auto_process_done);
+      return 0;
+    }
+    typedef int(*mi_crt_callback_t)(void);
+    #if defined(_WIN64)
+      #pragma comment(linker, "/INCLUDE:_mi_tls_callback")
+      #pragma section(".CRT$XIU", long, read)
+    #else
+      #pragma comment(linker, "/INCLUDE:__mi_tls_callback")
+    #endif
+    #pragma data_seg(".CRT$XIU")
+    mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach };
+    #pragma data_seg()
+  #endif
+
+  // use the fiber api for calling `_mi_thread_done`.
+  #include <fibersapi.h>
+  #if (_WIN32_WINNT < 0x600)  // before Windows Vista
+  WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );
+  WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex );
+  WINBASEAPI BOOL  WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData );
+  WINBASEAPI BOOL  WINAPI FlsFree(_In_ DWORD dwFlsIndex);
+  #endif
+
+  static DWORD mi_fls_key = (DWORD)(-1);
+
+  static void NTAPI mi_fls_done(PVOID value) {
+    mi_theap_t* theap = (mi_theap_t*)value;
+    if (theap != NULL) {
+      _mi_thread_done(theap);
+      FlsSetValue(mi_fls_key, NULL);  // prevent recursion as _mi_thread_done may set it back to the main theap, issue #672
+    }
+  }
+
+  void _mi_prim_thread_init_auto_done(void) {
+    mi_fls_key = FlsAlloc(&mi_fls_done);
+  }
+
+  void _mi_prim_thread_done_auto_done(void) {
+    // call thread-done on all threads (except the main thread) to prevent
+    // dangling callback pointer if statically linked with a DLL; Issue #208
+    FlsFree(mi_fls_key);
+  }
+
+  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
+    mi_assert_internal(mi_fls_key != (DWORD)(-1));
+    FlsSetValue(mi_fls_key, theap);
+  }
+#else
+#error "define windows process and thread auto initialization"
+#endif
+
+// ----------------------------------------------------
+// Communicate with the redirection module on Windows
+// ----------------------------------------------------
+#if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT)
+  #define MI_PRIM_HAS_ALLOCATOR_INIT 1
+
+  static bool mi_redirected = false;   // true if malloc redirects to mi_malloc
+
+  bool _mi_is_redirected(void) {
+    return mi_redirected;
+  }
+
+  #ifdef __cplusplus
+  extern "C" {
+  #endif
+  mi_decl_export void _mi_redirect_entry(DWORD reason) {
+    // called on redirection; careful as this may be called before DllMain
+    if (reason == DLL_PROCESS_ATTACH) {
+      mi_redirected = true;
+    }
+    else if (reason == DLL_PROCESS_DETACH) {
+      mi_redirected = false;
+    }
+    else if (reason == DLL_THREAD_DETACH) {
+      // mi_debug_out("redirect thread detach\n");
+      _mi_thread_done(NULL);
+    }
+  }
+  __declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message);
+  __declspec(dllimport) void mi_cdecl mi_allocator_done(void);
+  #ifdef __cplusplus
+  }
+  #endif
+  bool _mi_allocator_init(const char** message) {
+    return mi_allocator_init(message);
+  }
+  void _mi_allocator_done(void) {
+    mi_allocator_done();
+  }
+#endif
+
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/readme.md b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/readme.md
new file mode 100644
index 00000000..217c3d17
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/readme.md
@@ -0,0 +1,17 @@
+## Primitives:
+
+- `prim.c` contains Windows primitives for OS allocation.
+
+## Event Tracing for Windows (ETW)
+
+- `etw.h` is generated from `etw.man` which contains the manifest for mimalloc events.
+  (100 is an allocation, 101 is for a free)
+
+- `etw-mimalloc.wprp` is a profile for the Windows Performance Recorder (WPR).
+  In an admin prompt, you can use:
+  ```
+  > wpr -start src\prim\windows\etw-mimalloc.wprp -filemode
+  > <my mimalloc program>
+  > wpr -stop test.etl
+  ``` 
+  and then open `test.etl` in the Windows Performance Analyzer (WPA).
\ No newline at end of file
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/random.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/random.c
new file mode 100644
index 00000000..75b5cfbb
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/random.c
@@ -0,0 +1,258 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2019-2021, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"    // _mi_prim_random_buf
+
+/* ----------------------------------------------------------------------------
+We use our own PRNG to keep predictable performance of random number generation
+and to avoid implementations that use a lock. We only use the OS provided
+random source to initialize the initial seeds. Since we do not need ultimate
+performance but we do rely on the security (for secret cookies in secure mode)
+we use a cryptographically secure generator (chacha20).
+-----------------------------------------------------------------------------*/
+
+#define MI_CHACHA_ROUNDS (20)   // perhaps use 12 for better performance?
+
+
+/* ----------------------------------------------------------------------------
+Chacha20 implementation as the original algorithm with a 64-bit nonce
+and counter: https://en.wikipedia.org/wiki/Salsa20
+The input matrix has sixteen 32-bit values:
+Position  0 to  3: constant key
+Position  4 to 11: the key
+Position 12 to 13: the counter.
+Position 14 to 15: the nonce.
+
+The implementation uses regular C code which compiles very well on modern compilers.
+(gcc x64 has no register spills, and clang 6+ uses SSE instructions)
+-----------------------------------------------------------------------------*/
+
+static inline void qround(uint32_t x[16], size_t a, size_t b, size_t c, size_t d) {
+  x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 16);
+  x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 12);
+  x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 8);
+  x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 7);
+}
+
+static void chacha_block(mi_random_ctx_t* ctx)
+{
+  // scramble into `x`
+  uint32_t x[16];
+  for (size_t i = 0; i < 16; i++) {
+    x[i] = ctx->input[i];
+  }
+  for (size_t i = 0; i < MI_CHACHA_ROUNDS; i += 2) {
+    qround(x, 0, 4,  8, 12);
+    qround(x, 1, 5,  9, 13);
+    qround(x, 2, 6, 10, 14);
+    qround(x, 3, 7, 11, 15);
+    qround(x, 0, 5, 10, 15);
+    qround(x, 1, 6, 11, 12);
+    qround(x, 2, 7,  8, 13);
+    qround(x, 3, 4,  9, 14);
+  }
+
+  // add scrambled data to the initial state
+  for (size_t i = 0; i < 16; i++) {
+    ctx->output[i] = x[i] + ctx->input[i];
+  }
+  ctx->output_available = 16;
+
+  // increment the counter for the next round
+  ctx->input[12] += 1;
+  if (ctx->input[12] == 0) {
+    ctx->input[13] += 1;
+    if (ctx->input[13] == 0) {  // and keep increasing into the nonce
+      ctx->input[14] += 1;
+    }
+  }
+}
+
+static uint32_t chacha_next32(mi_random_ctx_t* ctx) {
+  if (ctx->output_available <= 0) {
+    chacha_block(ctx);
+    ctx->output_available = 16; // (assign again to suppress static analysis warning)
+  }
+  const uint32_t x = ctx->output[16 - ctx->output_available];
+  ctx->output[16 - ctx->output_available] = 0; // reset once the data is handed out
+  ctx->output_available--;
+  return x;
+}
+
+static inline uint32_t read32(const uint8_t* p, size_t idx32) {
+  const size_t i = 4*idx32;
+  return ((uint32_t)p[i+0] | (uint32_t)p[i+1] << 8 | (uint32_t)p[i+2] << 16 | (uint32_t)p[i+3] << 24);
+}
+
+static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t nonce)
+{
+  // since we only use chacha for randomness (and not encryption) we
+  // do not _need_ to read 32-bit values as little endian but we do anyways
+  // just for being compatible :-)
+  ctx->output_available = 0;
+  _mi_memzero(ctx->output,sizeof(ctx->output));
+  for (size_t i = 0; i < 4; i++) {
+    const uint8_t* sigma = (uint8_t*)"expand 32-byte k";
+    ctx->input[i] = read32(sigma,i);
+  }
+  for (size_t i = 0; i < 8; i++) {
+    ctx->input[i + 4] = read32(key,i);
+  }
+  ctx->input[12] = 0;
+  ctx->input[13] = 0;
+  ctx->input[14] = (uint32_t)nonce;
+  ctx->input[15] = (uint32_t)(nonce >> 32);
+}
+
+static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) {
+  _mi_memzero(ctx_new, sizeof(*ctx_new));
+  ctx_new->weak = ctx->weak;
+  _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input));
+  ctx_new->input[12] = 0;
+  ctx_new->input[13] = 0;
+  ctx_new->input[14] = (uint32_t)nonce;
+  ctx_new->input[15] = (uint32_t)(nonce >> 32);
+  mi_assert_internal(ctx->input[14] != ctx_new->input[14] || ctx->input[15] != ctx_new->input[15]); // do not reuse nonces!
+  chacha_block(ctx_new);
+}
+
+
+/* ----------------------------------------------------------------------------
+Random interface
+-----------------------------------------------------------------------------*/
+
+#if MI_DEBUG>1
+static bool mi_random_is_initialized(mi_random_ctx_t* ctx) {
+  return (ctx != NULL && ctx->input[0] != 0);
+}
+#endif
+
+void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) {
+  mi_assert_internal(mi_random_is_initialized(ctx));
+  mi_assert_internal(ctx != ctx_new);
+  chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new);
+}
+
+uintptr_t _mi_random_next(mi_random_ctx_t* ctx) {
+  mi_assert_internal(mi_random_is_initialized(ctx));
+  uintptr_t r;
+  do {
+    #if MI_INTPTR_SIZE <= 4
+    r = chacha_next32(ctx);
+    #elif MI_INTPTR_SIZE == 8
+    r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));
+    #else
+    # error "define mi_random_next for this platform"
+    #endif
+  } while (r==0);
+  return r;
+}
+
+
+/* ----------------------------------------------------------------------------
+To initialize a fresh random context.
+If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR.
+-----------------------------------------------------------------------------*/
+
+uintptr_t _mi_os_random_weak(uintptr_t extra_seed) {
+  uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random
+  x ^= _mi_prim_clock_now();
+  // and do a few randomization steps
+  uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1;
+  for (uintptr_t i = 0; i < max || x==0; i++, x++) {
+    x = _mi_random_shuffle(x);
+  }
+  mi_assert_internal(x != 0);
+  return x;
+}
+
+static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {
+  uint8_t key[32];
+  if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) {
+    // if we fail to get random data from the OS, we fall back to a
+    // weak random source based on the current time
+    #if !defined(__wasi__)
+    if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); }
+    #endif
+    uintptr_t x = _mi_os_random_weak(0);
+    for (size_t i = 0; i < 32; i+=4, x++) {  
+      x = _mi_random_shuffle(x);
+      key[i]   = (uint8_t)(x);
+      key[i+1] = (uint8_t)(x>>8);
+      key[i+2] = (uint8_t)(x>>16);
+      key[i+3] = (uint8_t)(x>>24);      
+    }
+    ctx->weak = true;
+  }
+  else {
+    ctx->weak = false;
+  }
+  chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ );
+}
+
+void _mi_random_init(mi_random_ctx_t* ctx) {
+  mi_random_init_ex(ctx, false);
+}
+
+void _mi_random_init_weak(mi_random_ctx_t * ctx) {
+  mi_random_init_ex(ctx, true);
+}
+
+void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx) {
+  if (ctx->weak) {
+    _mi_random_init(ctx);
+  }
+}
+
+/* --------------------------------------------------------
+test vectors from <https://tools.ietf.org/html/rfc8439>
+----------------------------------------------------------- */
+/*
+static bool array_equals(uint32_t* x, uint32_t* y, size_t n) {
+  for (size_t i = 0; i < n; i++) {
+    if (x[i] != y[i]) return false;
+  }
+  return true;
+}
+static void chacha_test(void)
+{
+  uint32_t x[4] = { 0x11111111, 0x01020304, 0x9b8d6f43, 0x01234567 };
+  uint32_t x_out[4] = { 0xea2a92f4, 0xcb1cf8ce, 0x4581472e, 0x5881c4bb };
+  qround(x, 0, 1, 2, 3);
+  mi_assert_internal(array_equals(x, x_out, 4));
+
+  uint32_t y[16] = {
+       0x879531e0,  0xc5ecf37d,  0x516461b1,  0xc9a62f8a,
+       0x44c20ef3,  0x3390af7f,  0xd9fc690b,  0x2a5f714c,
+       0x53372767,  0xb00a5631,  0x974c541a,  0x359e9963,
+       0x5c971061,  0x3d631689,  0x2098d9d6,  0x91dbd320 };
+  uint32_t y_out[16] = {
+       0x879531e0,  0xc5ecf37d,  0xbdb886dc,  0xc9a62f8a,
+       0x44c20ef3,  0x3390af7f,  0xd9fc690b,  0xcfacafd2,
+       0xe46bea80,  0xb00a5631,  0x974c541a,  0x359e9963,
+       0x5c971061,  0xccc07c79,  0x2098d9d6,  0x91dbd320 };
+  qround(y, 2, 7, 8, 13);
+  mi_assert_internal(array_equals(y, y_out, 16));
+
+  mi_random_ctx_t r = {
+    { 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574,
+      0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,
+      0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c,
+      0x00000001, 0x09000000, 0x4a000000, 0x00000000 },
+    {0},
+    0
+  };
+  uint32_t r_out[16] = {
+       0xe4e7f110, 0x15593bd1, 0x1fdd0f50, 0xc47120a3,
+       0xc7f4d1c7, 0x0368c033, 0x9aaa2204, 0x4e6cd4c3,
+       0x466482d2, 0x09aa9f07, 0x05d7c214, 0xa2028bd9,
+       0xd19c12b5, 0xb94e16de, 0xe883d0cb, 0x4e3c50a2 };
+  chacha_block(&r);
+  mi_assert_internal(array_equals(r.output, r_out, 16));
+}
+*/
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/static.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/static.c
new file mode 100644
index 00000000..2383f659
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/static.c
@@ -0,0 +1,43 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#ifndef _DEFAULT_SOURCE
+#define _DEFAULT_SOURCE
+#endif
+#if defined(__sun)
+// same remarks as os.c for the static's context.
+#undef _XOPEN_SOURCE
+#undef _POSIX_C_SOURCE
+#endif
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+
+// For a static override we create a single object file
+// containing the whole library. If it is linked first
+// it will override all the standard library allocation
+// functions (on Unix's).
+#include "alloc.c"          // includes alloc-override.c and free.c
+#include "alloc-aligned.c"
+#include "alloc-posix.c"
+#include "arena.c"
+#include "arena-meta.c"
+#include "bitmap.c"
+#include "heap.c"
+#include "init.c"
+#include "libc.c"
+#include "options.c"
+#include "os.c"
+#include "page.c"           // includes page-queue.c
+#include "page-map.c"
+#include "random.c"
+#include "stats.c"
+#include "theap.c"
+#include "threadlocal.c"
+#include "prim/prim.c"
+#if MI_OSX_ZONE
+#include "prim/osx/alloc-override-zone.c"
+#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/stats.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/stats.c
new file mode 100644
index 00000000..7ca18e23
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/stats.c
@@ -0,0 +1,813 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#include "mimalloc.h"
+#include "mimalloc-stats.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/atomic.h"
+#include "mimalloc/prim.h"
+
+#include <string.h> // memset
+
+#if defined(_MSC_VER) && (_MSC_VER < 1920)
+#pragma warning(disable:4204)  // non-constant aggregate initializer
+#endif
+
+/* -----------------------------------------------------------
+  Statistics operations
+----------------------------------------------------------- */
+
+static void mi_stat_update_mt(mi_stat_count_t* stat, int64_t amount) {
+  if (amount == 0) return;
+  // add atomically
+  int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount);
+  mi_atomic_maxi64_relaxed(&stat->peak, current + amount);
+  if (amount > 0) {
+    mi_atomic_addi64_relaxed(&stat->total, amount);
+  }
+}
+
+static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
+  if (amount == 0) return;
+  // add thread local
+  stat->current += amount;
+  if (stat->current > stat->peak) { stat->peak = stat->current; }
+  if (amount > 0) { stat->total += amount; }
+}
+
+
+void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) {
+  mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount);
+}
+
+void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
+  stat->total += amount;
+}
+
+void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) {
+  mi_stat_update_mt(stat, (int64_t)amount);
+}
+void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) {
+  mi_stat_update(stat, (int64_t)amount);
+}
+
+void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) {
+  mi_stat_update_mt(stat, -((int64_t)amount));
+}
+void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {
+  mi_stat_update(stat, -((int64_t)amount));
+}
+
+
+// Adjust stats to compensate; for example before committing a range,
+// first adjust downwards with parts that were already committed so
+// we avoid double counting.
+static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) {
+  if (amount == 0) return;
+  // adjust atomically
+  mi_atomic_addi64_relaxed(&stat->current, amount);
+  mi_atomic_addi64_relaxed(&stat->total, amount);
+}
+
+static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) {
+  if (amount == 0) return;
+  stat->current += amount;
+  stat->total += amount;
+}
+
+void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) {
+  mi_stat_adjust_mt(stat, (int64_t)amount);
+}
+void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) {
+  mi_stat_adjust(stat, (int64_t)amount);
+}
+void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) {
+  mi_stat_adjust_mt(stat, -((int64_t)amount));
+}
+void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {
+  mi_stat_adjust(stat, -((int64_t)amount));
+}
+
+
+// must be thread safe as it is called from stats_merge
+static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) {
+  if (stat==src) return;
+  mi_atomic_void_addi64_relaxed(&stat->total, &src->total);
+  const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current);
+
+  // Global current plus thread peak approximates new global peak
+  // note: peak scores do really not work across threads.
+  // we used to just add them together but that often overestimates in practice.
+  // similarly, max does not seem to work well. The current approach
+  // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112
+  // for a longer description.
+  mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak);
+}
+
+static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) {
+  if (stat==src) return;
+  mi_atomic_void_addi64_relaxed(&stat->total, &src->total);
+}
+
+#define MI_STAT_COUNT(stat)    mi_stat_count_add_mt(&stats->stat, &src->stat);
+#define MI_STAT_COUNTER(stat)  mi_stat_counter_add_mt(&stats->stat, &src->stat);
+
+// must be thread safe as it is called from stats_merge
+static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
+  if (stats==NULL || src==NULL || stats==src) return;
+
+  // copy all fields
+  MI_STAT_FIELDS()
+
+  #if MI_STAT>1
+  for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
+    mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]);
+  }
+  #endif
+  for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
+    mi_stat_count_add_mt(&stats->page_bins[i], &src->page_bins[i]);
+  }
+}
+
+#undef MI_STAT_COUNT
+#undef MI_STAT_COUNTER
+
+/* -----------------------------------------------------------
+  Display statistics
+----------------------------------------------------------- */
+
+// unit > 0 : size in binary bytes
+// unit == 0: count as decimal
+// unit < 0 : count in binary
+static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) {
+  char buf[32]; _mi_memzero_var(buf);
+  int  len = 32;
+  const char* suffix = (unit <= 0 ? " " : "B");
+  const int64_t base = (unit == 0 ? 1000 : 1024);
+  if (unit>0) n *= unit;
+
+  const int64_t pos = (n < 0 ? -n : n);
+  if (pos < base) {
+    if (n!=1 || suffix[0] != 'B') {  // skip printing 1 B for the unit column
+      _mi_snprintf(buf, len, "%lld   %-3s", (long long)n, (n==0 ? "" : suffix));
+    }
+  }
+  else {
+    int64_t divider = base;
+    const char* magnitude = "K";
+    if (pos >= divider*base) { divider *= base; magnitude = "M"; }
+    if (pos >= divider*base) { divider *= base; magnitude = "G"; }
+    const int64_t tens = (n / (divider/10));
+    const long whole = (long)(tens/10);
+    const long frac1 = (long)(tens%10);
+    char unitdesc[8];
+    _mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix);
+    _mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);
+  }
+  _mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf);
+}
+
+
+static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
+  mi_printf_amount(n,unit,out,arg,NULL);
+}
+
+static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
+  if (unit==1) _mi_fprintf(out, arg, "%12s"," ");
+          else mi_print_amount(n,0,out,arg);
+}
+
+static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) {
+  _mi_fprintf(out, arg,"  %-10s:", msg);
+  if (unit != 0) {
+    if (unit > 0) {
+      mi_print_amount(stat->peak, unit, out, arg);
+      mi_print_amount(stat->total, unit, out, arg);
+      // mi_print_amount(stat->freed, unit, out, arg);
+      mi_print_amount(stat->current, unit, out, arg);
+      mi_print_amount(unit, 1, out, arg);
+      mi_print_count(stat->total, unit, out, arg);
+    }
+    else {
+      mi_print_amount(stat->peak, -1, out, arg);
+      mi_print_amount(stat->total, -1, out, arg);
+      // mi_print_amount(stat->freed, -1, out, arg);
+      mi_print_amount(stat->current, -1, out, arg);
+      if (unit == -1) {
+        _mi_fprintf(out, arg, "%24s", "");
+      }
+      else {
+        mi_print_amount(-unit, 1, out, arg);
+        mi_print_count((stat->total / -unit), 0, out, arg);
+      }
+    }
+    if (stat->current != 0) {
+      _mi_fprintf(out, arg, "  ");
+      _mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok));
+      _mi_fprintf(out, arg, "\n");
+    }
+    else {
+      _mi_fprintf(out, arg, "  ok\n");
+    }
+  }
+  else {
+    mi_print_amount(stat->peak, 0, out, arg);
+    mi_print_amount(stat->total, 0, out, arg);
+    mi_print_amount(stat->current, 0, out, arg);
+    _mi_fprintf(out, arg, "\n");
+  }
+}
+
+static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
+  mi_stat_print_ex(stat, msg, unit, out, arg, NULL);
+}
+
+#if MI_STAT>1
+static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
+  _mi_fprintf(out, arg, "  %-10s:", msg);
+  _mi_fprintf(out, arg, "%12s", " ");  // no peak
+  mi_print_amount(stat->total, unit, out, arg);
+  _mi_fprintf(out, arg, "\n");
+}
+#endif
+
+static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
+  _mi_fprintf(out, arg, "  %-10s:", msg);
+  mi_print_amount(stat->total, 0, out, arg);
+  _mi_fprintf(out, arg, "\n");
+}
+
+static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
+  _mi_fprintf(out, arg, "  %-10s:", msg);
+  mi_print_amount(stat->total, 1, out, arg);
+  _mi_fprintf(out, arg, "\n");
+}
+
+static void mi_stat_average_print(int64_t count, int64_t total, const char* msg, mi_output_fun* out, void* arg) {
+  const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count));
+  const int64_t avg_whole = avg_tens/10;
+  const int64_t avg_frac1 = avg_tens%10;
+  _mi_fprintf(out, arg, "  %-10s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1);
+}
+
+
+static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) {
+  _mi_fprintf(out, arg, " %-11s %11s %11s %11s %11s %11s\n",
+                        name, "peak   ", "total   ", "current   ", "block   ", "total#   ");
+}
+
+#if MI_STAT>1
+static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) {
+  bool found = false;
+  char buf[64];
+  for (size_t i = 0; i <= max; i++) {
+    if (bins[i].total > 0) {
+      found = true;
+      const size_t unit = _mi_bin_size((uint8_t)i);
+      const char* pagekind = (unit <= MI_SMALL_MAX_OBJ_SIZE ? "S" :
+                               (unit <= MI_MEDIUM_MAX_OBJ_SIZE ? "M" :
+                                 (unit <= MI_LARGE_MAX_OBJ_SIZE ? "L" : "H")));
+      _mi_snprintf(buf, 64, "bin%2s  %3lu", pagekind, (long)i);
+      mi_stat_print(&bins[i], buf, (int64_t)unit, out, arg);
+    }
+  }
+  if (found) {
+    _mi_fprintf(out, arg, "\n");
+  }
+  return found;
+}
+#endif
+
+
+
+//------------------------------------------------------------
+// Use an output wrapper for line-buffered output
+// (which is nice when using loggers etc.)
+//------------------------------------------------------------
+typedef struct buffered_s {
+  mi_output_fun* out;   // original output function
+  void*          arg;   // and state
+  char*          buf;   // local buffer of at least size `count+1`
+  size_t         used;  // currently used chars `used <= count`
+  size_t         count; // total chars available for output
+} buffered_t;
+
+static void mi_buffered_flush(buffered_t* buf) {
+  buf->buf[buf->used] = 0;
+  _mi_fputs(buf->out, buf->arg, NULL, buf->buf);
+  buf->used = 0;
+}
+
+static void mi_cdecl mi_buffered_out(const char* msg, void* arg) {
+  buffered_t* buf = (buffered_t*)arg;
+  if (msg==NULL || buf==NULL) return;
+  for (const char* src = msg; *src != 0; src++) {
+    char c = *src;
+    if (buf->used >= buf->count) mi_buffered_flush(buf);
+    mi_assert_internal(buf->used < buf->count);
+    buf->buf[buf->used++] = c;
+    if (c == '\n') mi_buffered_flush(buf);
+  }
+}
+
+//------------------------------------------------------------
+// Print statistics
+//------------------------------------------------------------
+
+mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept
+{
+  size_t elapsed;
+  size_t user_time;
+  size_t sys_time;
+  size_t current_rss;
+  size_t peak_rss;
+  size_t current_commit;
+  size_t peak_commit;
+  size_t page_faults;
+  mi_process_info(&elapsed, &user_time, &sys_time, &current_rss, &peak_rss, &current_commit, &peak_commit, &page_faults);
+  _mi_fprintf(out, arg, "  %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000);
+  _mi_fprintf(out, arg, "  %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process",
+    user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults);
+  mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s");
+  if (peak_commit > 0) {
+    _mi_fprintf(out, arg, ", peak commit: ");
+    mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s");
+  }
+  _mi_fprintf(out, arg, "\n");
+}
+
+void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept {
+  // wrap the output function to be line buffered
+  char buf[256]; _mi_memzero_var(buf);
+  buffered_t buffer = { out0, arg0, NULL, 0, 255 };
+  buffer.buf = buf;
+  mi_output_fun* out = &mi_buffered_out;
+  void* arg = &buffer;
+
+  // and print using that
+  _mi_fprintf(out, arg, "%s %zu\n", name, id);
+
+  if (stats->malloc_normal.total + stats->malloc_huge.total != 0) {
+    #if MI_STAT>1
+    mi_print_header("blocks", out, arg);
+    mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg);
+    #endif
+    #if MI_STAT
+    mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg);
+    mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg);
+    mi_stat_count_t total = { 0,0,0 };
+    mi_stat_count_add_mt(&total, &stats->malloc_normal);
+    mi_stat_count_add_mt(&total, &stats->malloc_huge);
+    mi_stat_print_ex(&total, "total", 1, out, arg, "");
+    #if MI_STAT>1
+    mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg);
+    #endif
+    _mi_fprintf(out, arg, "\n");
+    #endif
+  }
+
+  if (stats->pages.total != 0) {
+    mi_print_header("pages", out, arg);
+    mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, "");
+    // mi_stat_print(&stats->segments, "segments", -1, out, arg);
+    // mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg);
+    // mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg);
+    mi_stat_print(&stats->pages, "pages", 0, out, arg);
+    mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg);
+    mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg);
+    mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg);
+    mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg);
+    mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg);
+    mi_stat_counter_print(&stats->pages_extended, "extended", out, arg);
+    mi_stat_counter_print(&stats->pages_retire, "retire", out, arg);
+    mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg);
+    _mi_fprintf(out, arg, "\n");
+  }
+
+  if (stats->arena_count.total > 0) {
+    mi_print_header("arenas", out, arg);
+    mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, "");
+    mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, "");
+    mi_stat_counter_print_size(&stats->reset, "reset", out, arg);
+    mi_stat_counter_print_size(&stats->purged, "purged", out, arg);
+
+    mi_stat_counter_print(&stats->arena_count, "arenas", out, arg);
+    mi_stat_counter_print(&stats->arena_rollback_count, "rollback", out, arg);
+    mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg);
+    mi_stat_counter_print(&stats->commit_calls, "commits", out, arg);
+    mi_stat_counter_print(&stats->reset_calls, "resets", out, arg);
+    mi_stat_counter_print(&stats->purge_calls, "purges", out, arg);
+    mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg);
+    mi_stat_print_ex(&stats->theaps, "theaps", 0, out, arg, "");
+    mi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, "");
+    mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg);
+    _mi_fprintf(out, arg, "\n");
+
+    mi_print_header("process", out, arg);
+    mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, "");
+    _mi_fprintf(out, arg, "  %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count());
+    mi_process_info_print_out(out, arg);
+  }
+  _mi_fprintf(out, arg, "\n");
+}
+
+
+static mi_msecs_t mi_process_start; // = 0
+
+// called on process init
+void _mi_stats_init(void) {
+  if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); };
+}
+
+static void mi_stats_add_into(mi_stats_t* to, const mi_stats_t* from) {
+  mi_assert_internal(to != NULL && from != NULL);
+  if (to == from) return;
+  mi_stats_add(to, from);
+}
+
+void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) {
+  mi_assert_internal(to != NULL && from != NULL);
+  if (to == from) return;
+  mi_stats_add(to, from);
+  _mi_memzero(from, sizeof(mi_stats_t));
+}
+
+static const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept {
+  mi_stats_t* stats = &theap->stats;
+  mi_stats_t* heap_stats = &_mi_theap_heap(theap)->stats;
+  _mi_stats_merge_into( heap_stats, stats );
+  return heap_stats;
+}
+
+static const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) {
+  if (heap==NULL) { heap = mi_heap_main(); }
+  mi_theap_t* theap = _mi_heap_theap_peek(heap);
+  if (theap==NULL) return &heap->stats;
+              else return mi_stats_merge_theap_to_heap(theap);
+}
+
+// deprecated
+void mi_stats_reset(void) mi_attr_noexcept {
+  if (!mi_theap_is_initialized(_mi_theap_default())) return;
+  mi_heap_get_stats(mi_heap_main());
+  mi_heap_stats_merge_to_subproc(mi_heap_main());
+}
+
+
+void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept {
+  if (heap==NULL) { heap = mi_heap_main(); }
+  _mi_stats_print("heap", heap->heap_seq, mi_heap_get_stats(heap), out, arg);
+}
+
+typedef struct mi_heap_print_visit_info_s {
+  mi_output_fun* out;
+  void* out_arg;
+} mi_heap_print_visit_info_t;
+
+static bool mi_cdecl mi_heap_print_visitor(mi_heap_t* heap, void* arg) {
+  mi_heap_print_visit_info_t* vinfo = (mi_heap_print_visit_info_t*)(arg);
+  mi_heap_stats_print_out(heap, vinfo->out, vinfo->out_arg);
+  return true;
+}
+
+
+// show each heap and then the subproc
+void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept {
+  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+  if (subproc==NULL) return;
+  mi_heap_print_visit_info_t vinfo = { out, arg };
+  mi_subproc_visit_heaps(subproc_id, &mi_heap_print_visitor, &vinfo);
+  _mi_stats_print("subproc", subproc->subproc_seq, &subproc->stats, out, arg);
+}
+
+
+// aggregate all stats from the heaps and subproc and print those
+void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept {
+  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+  if (subproc==NULL) return;
+  mi_stats_t_decl(stats);
+  if (mi_subproc_stats_get(subproc_id, &stats)) {
+    _mi_stats_print("subproc", subproc->subproc_seq, &stats, out, arg);
+  }
+}
+
+void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
+  mi_subproc_stats_print_out(mi_subproc_current(),out, arg);
+}
+
+// deprecated
+void mi_stats_print(void* out) mi_attr_noexcept {
+  // for compatibility there is an `out` parameter (which can be `stdout` or `stderr`)
+  mi_stats_print_out((mi_output_fun*)out, NULL);
+}
+
+// deprecated
+void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
+  mi_theap_t* theap = _mi_theap_default();
+  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
+  _mi_stats_print("heap", _mi_theap_heap(theap)->heap_seq, &theap->stats, out, arg);
+  mi_stats_merge_theap_to_heap(_mi_theap_default());
+}
+
+
+// ----------------------------------------------------------------
+// Basic timer for convenience; use milli-seconds to avoid doubles
+// ----------------------------------------------------------------
+
+static mi_msecs_t mi_clock_diff;
+
+mi_msecs_t _mi_clock_now(void) {
+  return _mi_prim_clock_now();
+}
+
+mi_msecs_t _mi_clock_start(void) {
+  if (mi_clock_diff == 0.0) {
+    mi_msecs_t t0 = _mi_clock_now();
+    mi_clock_diff = _mi_clock_now() - t0;
+  }
+  return _mi_clock_now();
+}
+
+mi_msecs_t _mi_clock_end(mi_msecs_t start) {
+  mi_msecs_t end = _mi_clock_now();
+  return (end - start - mi_clock_diff);
+}
+
+
+// --------------------------------------------------------
+// Basic process statistics
+// --------------------------------------------------------
+
+mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept
+{
+  mi_process_info_t pinfo;
+  _mi_memzero_var(pinfo);
+  pinfo.elapsed        = _mi_clock_end(mi_process_start);
+  { const mi_subproc_t* subproc = _mi_subproc_main();
+    if (subproc!=NULL) {
+      pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)));
+      pinfo.peak_commit    = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)));
+    }  
+  }
+  pinfo.current_rss    = pinfo.current_commit;
+  pinfo.peak_rss       = pinfo.peak_commit;
+  pinfo.utime          = 0;
+  pinfo.stime          = 0;
+  pinfo.page_faults    = 0;
+
+  _mi_prim_process_info(&pinfo);
+
+  if (elapsed_msecs!=NULL)  *elapsed_msecs  = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX));
+  if (user_msecs!=NULL)     *user_msecs     = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX));
+  if (system_msecs!=NULL)   *system_msecs   = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX));
+  if (current_rss!=NULL)    *current_rss    = pinfo.current_rss;
+  if (peak_rss!=NULL)       *peak_rss       = pinfo.peak_rss;
+  if (current_commit!=NULL) *current_commit = pinfo.current_commit;
+  if (peak_commit!=NULL)    *peak_commit    = pinfo.peak_commit;
+  if (page_faults!=NULL)    *page_faults    = pinfo.page_faults;
+}
+
+mi_decl_export void mi_process_info_print(void) mi_attr_noexcept {
+  mi_process_info_print_out(NULL, NULL);
+}
+
+
+// --------------------------------------------------------
+// Return statistics
+// --------------------------------------------------------
+
+size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept {
+  if (bin > MI_BIN_HUGE) return 0;
+  return _mi_bin_size(bin);
+}
+
+static bool mi_stats_copy(mi_stats_t* stats_to, const mi_stats_t* stats_from) mi_attr_noexcept {
+  if (stats_to == NULL || stats_to->size != sizeof(mi_stats_t) || stats_to->version != MI_STAT_VERSION) return false;
+  if (stats_from == NULL || stats_from->size != stats_to->size) return false;
+  _mi_memcpy(stats_to, stats_from, stats_to->size);
+  return true;
+}
+
+bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {
+  const mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+  if (subproc==NULL) return false;
+  return mi_stats_copy(stats,&subproc->stats);
+}
+
+bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept {
+  return mi_stats_copy(stats, mi_heap_get_stats(heap));
+}
+
+
+static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) {
+  mi_stats_t* stats = (mi_stats_t*)arg;
+  mi_stats_add_into(stats, mi_heap_get_stats(heap));
+  return true;
+}
+
+bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {
+  if (stats==NULL) return false;
+  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+  if (subproc == NULL) return false;
+  if (!mi_stats_copy(stats, &subproc->stats)) return false;
+  mi_subproc_visit_heaps(subproc_id, &mi_heap_aggregate_visitor, stats);
+  return true;
+}
+
+bool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept {
+  return mi_subproc_stats_get(mi_subproc_current(), stats);
+}
+
+
+// --------------------------------------------------------
+// Statics in json format
+// --------------------------------------------------------
+
+typedef struct mi_json_buf_s {
+  char*   buf;
+  size_t  size;
+  size_t  used;
+  bool    can_realloc;
+} mi_json_buf_t;
+
+static bool mi_json_buf_expand(mi_json_buf_t* hbuf) {
+  if (hbuf==NULL) return false;
+  if (hbuf->buf != NULL && hbuf->size>0) {
+    hbuf->buf[hbuf->size-1] = 0;
+  }
+  if (hbuf->size > SIZE_MAX/2 || !hbuf->can_realloc) return false;
+  const size_t newsize = (hbuf->size == 0 ? mi_good_size(12*MI_KiB) : 2*hbuf->size);
+  char* const  newbuf  = (char*)mi_rezalloc(hbuf->buf, newsize);
+  if (newbuf == NULL) return false;
+  hbuf->buf = newbuf;
+  hbuf->size = newsize;
+  return true;
+}
+
+static void mi_json_buf_print(mi_json_buf_t* hbuf, const char* msg) {
+  if (msg==NULL || hbuf==NULL) return;
+  if (hbuf->used + 1 >= hbuf->size && !hbuf->can_realloc) return;
+  for (const char* src = msg; *src != 0; src++) {
+    char c = *src;
+    if (hbuf->used + 1 >= hbuf->size) {
+      if (!mi_json_buf_expand(hbuf)) return;
+    }
+    mi_assert_internal(hbuf->used < hbuf->size);
+    hbuf->buf[hbuf->used++] = c;
+  }
+  mi_assert_internal(hbuf->used < hbuf->size);
+  hbuf->buf[hbuf->used] = 0;
+}
+
+static void mi_json_buf_print_count_bin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, size_t bin, bool add_comma) {
+  const size_t binsize = mi_stats_get_bin_size(bin);
+  const size_t pagesize = (binsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_SMALL_PAGE_SIZE :
+                            (binsize <= MI_MEDIUM_MAX_OBJ_SIZE ? MI_MEDIUM_PAGE_SIZE :
+                              (binsize <= MI_LARGE_MAX_OBJ_SIZE ? MI_LARGE_PAGE_SIZE : 0)));
+  char buf[128];
+  _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"block_size\": %zu, \"page_size\": %zu }%s\n", prefix, stat->total, stat->peak, stat->current, binsize, pagesize, (add_comma ? "," : ""));
+  buf[127] = 0;
+  mi_json_buf_print(hbuf, buf);
+}
+
+static void mi_json_buf_print_count_cbin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, mi_chunkbin_t bin, bool add_comma) {
+  const char* cbin = " ";
+  switch(bin) {
+    case MI_CBIN_SMALL:  cbin = "S"; break;
+    case MI_CBIN_MEDIUM: cbin = "M"; break;
+    case MI_CBIN_LARGE:  cbin = "L"; break;
+    case MI_CBIN_HUGE:   cbin = "H"; break;
+    case MI_CBIN_OTHER:  cbin = "X"; break;
+    default: cbin = " "; break;
+  }
+  char buf[128];
+  _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"bin\": \"%s\" }%s\n", prefix, stat->total, stat->peak, stat->current, cbin, (add_comma ? "," : ""));
+  buf[127] = 0;
+  mi_json_buf_print(hbuf, buf);
+}
+
+static void mi_json_buf_print_count(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, bool add_comma) {
+  char buf[128];
+  _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld }%s\n", prefix, stat->total, stat->peak, stat->current, (add_comma ? "," : ""));
+  buf[127] = 0;
+  mi_json_buf_print(hbuf, buf);
+}
+
+static void mi_json_buf_print_count_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_count_t* stat) {
+  char buf[128];
+  _mi_snprintf(buf, 128, "  \"%s\": ", name);
+  buf[127] = 0;
+  mi_json_buf_print(hbuf, buf);
+  mi_json_buf_print_count(hbuf, "", stat, true);
+}
+
+static void mi_json_buf_print_value(mi_json_buf_t* hbuf, const char* name, int64_t val) {
+  char buf[128];
+  _mi_snprintf(buf, 128, "  \"%s\": %lld,\n", name, val);
+  buf[127] = 0;
+  mi_json_buf_print(hbuf, buf);
+}
+
+static void mi_json_buf_print_size(mi_json_buf_t* hbuf, const char* name, size_t val, bool add_comma) {
+  char buf[128];
+  _mi_snprintf(buf, 128, "    \"%s\": %zu%s\n", name, val, (add_comma ? "," : ""));
+  buf[127] = 0;
+  mi_json_buf_print(hbuf, buf);
+}
+
+static void mi_json_buf_print_counter_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_counter_t* stat) {
+  mi_json_buf_print_value(hbuf, name, stat->total);
+}
+
+#define MI_STAT_COUNT(stat)    mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat);
+#define MI_STAT_COUNTER(stat)  mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat);
+
+static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, char* output_buf) mi_attr_noexcept {
+  if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL;
+  mi_json_buf_t hbuf = { NULL, 0, 0, true };
+  if (output_size > 0 && output_buf != NULL) {
+    _mi_memzero(output_buf, output_size);
+    hbuf.buf = output_buf;
+    hbuf.size = output_size;
+    hbuf.can_realloc = false;
+  }
+  else {
+    if (!mi_json_buf_expand(&hbuf)) return NULL;
+  }
+  mi_json_buf_print(&hbuf, "{\n");
+  mi_json_buf_print_value(&hbuf, "stat_version", MI_STAT_VERSION);
+  mi_json_buf_print_value(&hbuf, "mimalloc_version", MI_MALLOC_VERSION);
+
+  // process info
+  mi_json_buf_print(&hbuf, "  \"process\": {\n");
+  size_t elapsed;
+  size_t user_time;
+  size_t sys_time;
+  size_t current_rss;
+  size_t peak_rss;
+  size_t current_commit;
+  size_t peak_commit;
+  size_t page_faults;
+  mi_process_info(&elapsed, &user_time, &sys_time, &current_rss, &peak_rss, &current_commit, &peak_commit, &page_faults);
+  mi_json_buf_print_size(&hbuf, "elapsed_msecs", elapsed, true);
+  mi_json_buf_print_size(&hbuf, "user_msecs", user_time, true);
+  mi_json_buf_print_size(&hbuf, "system_msecs", sys_time, true);
+  mi_json_buf_print_size(&hbuf, "page_faults", page_faults, true);
+  mi_json_buf_print_size(&hbuf, "rss_current", current_rss, true);
+  mi_json_buf_print_size(&hbuf, "rss_peak", peak_rss, true);
+  mi_json_buf_print_size(&hbuf, "commit_current", current_commit, true);
+  mi_json_buf_print_size(&hbuf, "commit_peak", peak_commit, false);
+  mi_json_buf_print(&hbuf, "  },\n");
+
+  // statistics
+  MI_STAT_FIELDS()
+
+  // size bins
+  mi_json_buf_print(&hbuf, "  \"malloc_bins\": [\n");
+  for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
+    mi_json_buf_print_count_bin(&hbuf, "    ", &stats->malloc_bins[i], i, i!=MI_BIN_HUGE);
+  }
+  mi_json_buf_print(&hbuf, "  ],\n");
+  mi_json_buf_print(&hbuf, "  \"page_bins\": [\n");
+  for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
+    mi_json_buf_print_count_bin(&hbuf, "    ", &stats->page_bins[i], i, i!=MI_BIN_HUGE);
+  }
+  mi_json_buf_print(&hbuf, "  ],\n");
+  mi_json_buf_print(&hbuf, "  \"chunk_bins\": [\n");
+  for (size_t i = 0; i < MI_CBIN_COUNT; i++) {
+    mi_json_buf_print_count_cbin(&hbuf, "    ", &stats->chunk_bins[i], (mi_chunkbin_t)i, i!=MI_CBIN_COUNT-1);
+  }
+  mi_json_buf_print(&hbuf, "  ]\n");
+  mi_json_buf_print(&hbuf, "}\n");
+  if (hbuf.used >= hbuf.size) {
+    // failed
+    if (hbuf.can_realloc) { mi_free(hbuf.buf); }
+    return NULL;
+  }
+  else {
+    return hbuf.buf;
+  }
+}
+
+char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept {
+  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+  if (subproc==NULL) return NULL;
+  mi_stats_t_decl(stats);
+  if (!mi_subproc_stats_get(subproc_id,&stats)) return NULL;
+  return mi_stats_get_json_from(&stats, buf_size, buf);
+}
+
+char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept {
+  return mi_stats_get_json_from(mi_heap_get_stats(heap), buf_size, buf);
+}
+
+char* mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept {
+  return mi_subproc_stats_get_json(mi_subproc_current(), buf_size, buf);
+}
+
+char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept {
+  return mi_stats_get_json_from(stats, buf_size, buf);
+}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/theap.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/theap.c
new file mode 100644
index 00000000..ef53c1e0
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/theap.c
@@ -0,0 +1,715 @@
+/*----------------------------------------------------------------------------
+Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"  // _mi_theap_default
+
+#if defined(_MSC_VER) && (_MSC_VER < 1920)
+#pragma warning(disable:4204)  // non-constant aggregate initializer
+#endif
+
+/* -----------------------------------------------------------
+  Helpers
+----------------------------------------------------------- */
+
+// return `true` if ok, `false` to break
+typedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2);
+
+// Visit all pages in a theap; returns `false` if break was called.
+static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2)
+{
+  if (theap==NULL || theap->page_count==0) return 0;
+
+  // visit all pages
+  #if MI_DEBUG>1
+  size_t total = theap->page_count;
+  size_t count = 0;
+  #endif
+
+  const size_t max_bin = (include_full ? MI_BIN_FULL : MI_BIN_FULL - 1);
+  for (size_t i = 0; i <= max_bin; i++) {
+    mi_page_queue_t* pq = &theap->pages[i];
+    mi_page_t* page = pq->first;
+    while(page != NULL) {
+      mi_page_t* next = page->next; // save next in case the page gets removed from the queue
+      mi_assert_internal(mi_page_theap(page) == theap);
+      #if MI_DEBUG>1
+      count++;
+      #endif
+      if (!fn(theap, pq, page, arg1, arg2)) return false;
+      page = next; // and continue
+    }
+  }
+  mi_assert_internal(!include_full || count == total);
+  return true;
+}
+
+
+#if MI_DEBUG>=2
+static bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
+  MI_UNUSED(arg1);
+  MI_UNUSED(arg2);
+  MI_UNUSED(pq);
+  mi_assert_internal(mi_page_theap(page) == theap);
+  mi_assert_expensive(_mi_page_is_valid(page));
+  return true;
+}
+#endif
+#if MI_DEBUG>=3
+static bool mi_theap_is_valid(mi_theap_t* theap) {
+  mi_assert_internal(theap!=NULL);
+  mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL);
+  for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
+    mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin]));
+  }
+  return true;
+}
+#endif
+
+
+
+
+/* -----------------------------------------------------------
+  "Collect" pages by migrating `local_free` and `thread_free`
+  lists and freeing empty pages. This is done when a thread
+  stops (and in that case abandons pages if there are still
+  blocks alive)
+----------------------------------------------------------- */
+
+typedef enum mi_collect_e {
+  MI_NORMAL,
+  MI_FORCE,
+  MI_ABANDON
+} mi_collect_t;
+
+
+static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {
+  MI_UNUSED(arg2);
+  MI_UNUSED(theap);
+  mi_assert_internal(mi_theap_page_is_valid(theap, pq, page, NULL, NULL));
+  mi_collect_t collect = *((mi_collect_t*)arg_collect);
+  _mi_page_free_collect(page, collect >= MI_FORCE);
+  if (mi_page_all_free(page)) {
+    // no more used blocks, possibly free the page.
+    if (collect >= MI_FORCE || page->retire_expire == 0) {  // either forced/abandon, or not already retired
+      // note: this will potentially free retired pages as well.
+      _mi_page_free(page, pq);
+    }
+  }
+  else if (collect == MI_ABANDON) {
+    // still used blocks but the thread is done; abandon the page
+    _mi_page_abandon(page, pq);
+  }
+  return true; // don't break
+}
+
+static void mi_theap_merge_stats(mi_theap_t* theap) {
+  mi_assert_internal(mi_theap_is_initialized(theap));
+  _mi_stats_merge_into(&_mi_theap_heap(theap)->stats, &theap->stats);
+}
+
+static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect)
+{
+  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
+  mi_assert_expensive(mi_theap_is_valid(theap));
+
+  const bool force = (collect >= MI_FORCE);
+  _mi_deferred_free(theap, force);
+
+  // python/cpython#112532: we may be called from a thread that is not the owner of the theap
+  // const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id());
+
+  // collect retired pages
+  _mi_theap_collect_retired(theap, force);
+
+  // collect all pages owned by this thread
+  mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL);  // dont normally visit full pages, see issue #1220
+
+  // collect arenas (this is program wide so don't force purges on abandonment of threads)
+  //mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1);
+  _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld);
+
+  // merge statistics
+  mi_theap_merge_stats(theap);
+}
+
+void _mi_theap_collect_abandon(mi_theap_t* theap) {
+  mi_theap_collect_ex(theap, MI_ABANDON);
+}
+
+void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept {
+  mi_theap_collect_ex(theap, (force ? MI_FORCE : MI_NORMAL));
+}
+
+void mi_collect(bool force) mi_attr_noexcept {
+  // cannot really collect process wide, just a theap..
+  mi_theap_collect(_mi_theap_default(), force);
+}
+
+void mi_heap_collect(mi_heap_t* heap, bool force) {
+  // cannot really collect a heap, just a theap..
+  mi_theap_collect(mi_heap_theap(heap), force);
+}
+
+/* -----------------------------------------------------------
+  Heap new
+----------------------------------------------------------- */
+
+mi_theap_t* mi_theap_get_default(void) {
+  mi_theap_t* theap = _mi_theap_default();
+  if mi_unlikely(!mi_theap_is_initialized(theap)) {
+    mi_thread_init();
+    theap = _mi_theap_default();
+    mi_assert_internal(mi_theap_is_initialized(theap));
+  }
+  return theap;
+}
+
+// todo: make order of parameters consistent (but would that break compat with CPython?)
+void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)
+{
+  mi_assert_internal(theap!=NULL);
+  mi_assert_internal(heap!=NULL);
+  mi_memid_t memid = theap->memid;
+  _mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t));
+  theap->memid = memid;
+  theap->refcount = 1;
+  theap->tld   = tld;  // avoid reading the thread-local tld during initialization
+  mi_atomic_store_ptr_relaxed(mi_heap_t,&theap->heap,heap);
+  mi_assert_internal(theap->stats.size == sizeof(mi_stats_t));
+  
+  _mi_theap_options_init(theap);
+  if (theap->tld->is_in_threadpool) {
+    // if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap.
+    // this is checked in `free.c:mi_free_try_collect_mt`
+    // .. but abandoning is good in this case: halve the full page retain (possibly to 0)
+    // (so blocked threads do not hold on to too much memory)
+    if (theap->page_full_retain > 0) {
+      theap->page_full_retain = theap->page_full_retain / 4;
+    }
+  }
+
+  // push on the thread local theaps list
+  mi_theap_t* head = NULL;
+  mi_lock(&theap->tld->theaps_lock) {
+    head = theap->tld->theaps;
+    theap->tprev = NULL;
+    theap->tnext = head;
+    if (head!=NULL) { head->tprev = theap; }
+    theap->tld->theaps = theap;
+  }
+
+  // initialize random
+  if (head == NULL) {  // first theap in this thread?
+    #if defined(_WIN32) && !defined(MI_SHARED_LIB)
+      _mi_random_init_weak(&theap->random);    // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)
+    #else
+      _mi_random_init(&theap->random);
+    #endif
+  }
+  else {
+    _mi_random_split(&head->random, &theap->random);
+  }
+  theap->cookie  = _mi_theap_random_next(theap) | 1;
+  _mi_theap_guarded_init(theap);
+  mi_subproc_stat_increase(_mi_subproc(),theaps,1);
+
+  // push on the heap's theap list
+  mi_lock(&heap->theaps_lock) {
+    head = heap->theaps;
+    theap->hprev = NULL;
+    theap->hnext = head;
+    if (head!=NULL) { head->hprev = theap; }
+    heap->theaps = theap;
+  }
+}
+
+mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) {
+  mi_assert_internal(tld!=NULL);
+  mi_assert_internal(heap!=NULL);
+  // allocate and initialize a theap
+  mi_memid_t memid;
+  mi_theap_t* theap;
+  //if (!_mi_is_heap_main(heap)) {
+  //  theap = (mi_theap_t*)mi_heap_zalloc(mi_heap_main(),sizeof(mi_theap_t));
+  //  memid = _mi_memid_create(MI_MEM_HEAP_MAIN);
+  //  memid.initially_zero = memid.initially_committed = true;
+  //}
+  //else
+  if (heap->exclusive_arena == NULL) {
+    theap = (mi_theap_t*)_mi_meta_zalloc(sizeof(mi_theap_t), &memid);
+  }
+  else {
+    // theaps associated with a specific arena are allocated in that arena
+    // note: takes up at least one slice which is quite wasteful...
+    const size_t size = _mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE);
+    theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid);
+    mi_assert_internal(memid.mem.os.size >= size);
+  }
+  if (theap==NULL) {
+    _mi_error_message(ENOMEM, "unable to allocate theap meta-data\n");
+    return NULL;
+  }
+  theap->memid = memid;
+  _mi_theap_init(theap, heap, tld);
+  return theap;
+}
+
+uintptr_t _mi_theap_random_next(mi_theap_t* theap) {
+  return _mi_random_next(&theap->random);
+}
+
+static void mi_theap_free_mem(mi_theap_t* theap) {
+  if (theap!=NULL) {
+    mi_subproc_stat_decrease(_mi_subproc(),theaps,1);
+    // free the used memory
+    if (theap->memid.memkind == MI_MEM_HEAP_MAIN) {  // note: for now unused as it would access theap_default stats in mi_free of the current theap
+      mi_assert_internal(_mi_is_heap_main(mi_heap_of(theap)));
+      mi_free(theap);
+    }
+    else if (theap->memid.memkind == MI_MEM_META) {
+      _mi_meta_free(theap, sizeof(*theap), theap->memid);
+    }
+    else {
+      _mi_arenas_free(theap, _mi_align_up(sizeof(*theap),MI_ARENA_MIN_OBJ_SIZE), theap->memid ); // issue #1168, avoid assertion failure
+    }
+  }
+}
+
+void _mi_theap_incref(mi_theap_t* theap) {
+  if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {
+    mi_atomic_increment_acq_rel(&theap->refcount);
+  }
+}
+
+void _mi_theap_decref(mi_theap_t* theap) {
+  if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {
+    if (mi_atomic_decrement_acq_rel(&theap->refcount) == 1) {
+      mi_theap_free_mem(theap);
+    }
+  }
+}
+
+
+// called from `mi_theap_delete` to free the internal theap resources.
+bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock) {
+  mi_assert(theap != NULL);
+  if (theap==NULL) return true;
+
+  mi_heap_t* const heap = mi_atomic_exchange_ptr_acq_rel(mi_heap_t, &theap->heap, NULL);
+  if (heap==NULL) {
+    // concurrent interaction, retry in an outer loop (as the other thread may be blocked on our lock)
+    return false;
+  }
+  else {
+    // merge stats to the owning heap
+    _mi_stats_merge_into(&heap->stats, &theap->stats);
+
+    // remove ourselves from the heap theaps list
+    mi_lock_maybe(&heap->theaps_lock, acquire_heap_theaps_lock) {
+      if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; }
+      if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; }
+                          else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; }
+      theap->hnext = theap->hprev = NULL;
+    }
+
+    // remove ourselves from the thread local theaps list
+    mi_lock_maybe(&theap->tld->theaps_lock, acquire_tld_theaps_lock) {
+      if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev;  }
+      if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext;  }
+                          else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; }
+      theap->tnext = theap->tprev = NULL;                        
+    }
+    theap->tld = NULL;
+    _mi_theap_decref(theap);
+    return true;
+  }
+}
+
+
+/* -----------------------------------------------------------
+  Heap destroy
+----------------------------------------------------------- */
+/*
+
+// zero out the page queues
+static void mi_theap_reset_pages(mi_theap_t* theap) {
+  mi_assert_internal(theap != NULL);
+  mi_assert_internal(mi_theap_is_initialized(theap));
+  // TODO: copy full empty theap instead?
+  _mi_memset(&theap->pages_free_direct, 0, sizeof(theap->pages_free_direct));
+  _mi_memcpy_aligned(&theap->pages, &_mi_theap_empty.pages, sizeof(theap->pages));
+  // theap->thread_delayed_free = NULL;
+  theap->page_count = 0;
+}
+
+static bool _mi_theap_page_destroy(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
+  MI_UNUSED(arg1);
+  MI_UNUSED(arg2);
+  MI_UNUSED(pq);
+
+  // ensure no more thread_delayed_free will be added
+  //_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);
+
+  // stats
+  const size_t bsize = mi_page_block_size(page);
+  if (bsize > MI_LARGE_MAX_OBJ_SIZE) {
+    mi_theap_stat_decrease(theap, malloc_huge, bsize);
+  }
+  #if (MI_STAT>0)
+  _mi_page_free_collect(page, false);  // update used count
+  const size_t inuse = page->used;
+  if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
+    mi_theap_stat_decrease(theap, malloc_normal, bsize * inuse);
+    #if (MI_STAT>1)
+    mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], inuse);
+    #endif
+  }
+  // mi_theap_stat_decrease(theap, malloc_requested, bsize * inuse);  // todo: off for aligned blocks...
+  #endif
+
+  /// pretend it is all free now
+  mi_assert_internal(mi_page_thread_free(page) == NULL);
+  page->used = 0;
+
+  // and free the page
+  // mi_page_free(page,false);
+  page->next = NULL;
+  page->prev = NULL;
+  mi_page_set_theap(page, NULL);
+  _mi_arenas_page_free(page, theap);
+
+  return true; // keep going
+}
+
+void _mi_theap_destroy_pages(mi_theap_t* theap) {
+  mi_theap_visit_pages(theap, &_mi_theap_page_destroy, NULL, NULL);
+  mi_theap_reset_pages(theap);
+}
+
+#if MI_TRACK_HEAP_DESTROY
+static bool mi_cdecl mi_theap_track_block_free(const mi_theap_t* theap, const mi_theap_area_t* area, void* block, size_t block_size, void* arg) {
+  MI_UNUSED(theap); MI_UNUSED(area);  MI_UNUSED(arg); MI_UNUSED(block_size);
+  mi_track_free_size(block,mi_usable_size(block));
+  return true;
+}
+#endif
+
+void mi_theap_destroy(mi_theap_t* theap) {
+  mi_assert(theap != NULL);
+  mi_assert(mi_theap_is_initialized(theap));
+  mi_assert(!theap->allow_page_reclaim);
+  mi_assert(!theap->allow_page_abandon);
+  mi_assert_expensive(mi_theap_is_valid(theap));
+  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
+  #if MI_GUARDED
+  // _mi_warning_message("'mi_theap_destroy' called but MI_GUARDED is enabled -- using `mi_theap_delete` instead (theap at %p)\n", theap);
+  mi_theap_delete(theap);
+  return;
+  #else
+  if (theap->allow_page_reclaim) {
+    _mi_warning_message("'mi_theap_destroy' called but ignored as the theap was not created with 'allow_destroy' (theap at %p)\n", theap);
+    // don't free in case it may contain reclaimed pages,
+    mi_theap_delete(theap);
+  }
+  else {
+    // track all blocks as freed
+    #if MI_TRACK_HEAP_DESTROY
+    mi_theap_visit_blocks(theap, true, mi_theap_track_block_free, NULL);
+    #endif
+    // free all pages
+    _mi_theap_destroy_pages(theap);
+    mi_theap_free(theap,true);
+  }
+  #endif
+}
+
+// forcefully destroy all theaps in the current thread
+void _mi_theap_unsafe_destroy_all(mi_theap_t* theap) {
+  mi_assert_internal(theap != NULL);
+  if (theap == NULL) return;
+  mi_theap_t* curr = theap->tld->theaps;
+  while (curr != NULL) {
+    mi_theap_t* next = curr->next;
+    if (!curr->allow_page_reclaim) {
+      mi_theap_destroy(curr);
+    }
+    else {
+      _mi_theap_destroy_pages(curr);
+    }
+    curr = next;
+  }
+}
+*/
+
+/* -----------------------------------------------------------
+  Safe Heap delete
+----------------------------------------------------------- */
+
+// Safe delete a theap without freeing any still allocated blocks in that theap.
+void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock)
+{
+  mi_assert(theap != NULL);
+  mi_assert(mi_theap_is_initialized(theap));
+  mi_assert_expensive(mi_theap_is_valid(theap));
+  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
+
+  // abandon all pages
+  _mi_theap_collect_abandon(theap);
+
+  mi_assert_internal(theap->page_count==0);
+  _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock);
+}
+
+
+
+/* -----------------------------------------------------------
+  Load/unload theaps
+----------------------------------------------------------- */
+/*
+void mi_theap_unload(mi_theap_t* theap) {
+  mi_assert(mi_theap_is_initialized(theap));
+  mi_assert_expensive(mi_theap_is_valid(theap));
+  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
+  if (_mi_theap_heap(theap)->exclusive_arena == NULL) {
+    _mi_warning_message("cannot unload theaps that are not associated with an exclusive arena\n");
+    return;
+  }
+
+  // abandon all pages so all thread'id in the pages are cleared
+  _mi_theap_collect_abandon(theap);
+  mi_assert_internal(theap->page_count==0);
+
+  // remove from theap list
+  mi_theap_free(theap, false); // but don't actually free the memory
+
+  // disassociate from the current thread-local and static state
+  theap->tld = NULL;
+  return;
+}
+
+bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) {
+  mi_assert(mi_theap_is_initialized(theap));
+  if (theap==NULL || !mi_theap_is_initialized(theap)) return false;
+  if (_mi_theap_heap(theap)->exclusive_arena == NULL) {
+    _mi_warning_message("cannot reload theaps that were not associated with an exclusive arena\n");
+    return false;
+  }
+  if (theap->tld != NULL) {
+    _mi_warning_message("cannot reload theaps that were not unloaded first\n");
+    return false;
+  }
+  mi_arena_t* arena = _mi_arena_from_id(arena_id);
+  if (_mi_theap_heap(theap)->exclusive_arena != arena) {
+    _mi_warning_message("trying to reload a theap at a different arena address: %p vs %p\n", _mi_theap_heap(theap)->exclusive_arena, arena);
+    return false;
+  }
+
+  mi_assert_internal(theap->page_count==0);
+
+  // re-associate with the current thread-local and static state
+  theap->tld = mi_theap_get_default()->tld;
+
+  // reinit direct pages (as we may be in a different process)
+  mi_assert_internal(theap->page_count == 0);
+  for (size_t i = 0; i < MI_PAGES_DIRECT; i++) {
+    theap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty;
+  }
+
+  // push on the thread local theaps list
+  theap->tnext = theap->tld->theaps;
+  theap->tld->theaps = theap;
+  return true;
+}
+*/
+
+
+/* -----------------------------------------------------------
+  Visit all theap blocks and areas
+  Todo: enable visiting abandoned pages, and
+        enable visiting all blocks of all theaps across threads
+----------------------------------------------------------- */
+
+void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {
+  const size_t bsize = mi_page_block_size(page);
+  const size_t ubsize = mi_page_usable_block_size(page);
+  area->reserved = page->reserved * bsize;
+  area->committed = page->capacity * bsize;
+  area->blocks = mi_page_start(page);
+  area->used = page->used;   // number of blocks in use (#553)
+  area->block_size = ubsize;
+  area->full_block_size = bsize;
+  area->reserved1 = page;
+}
+
+static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) {
+  mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX);
+  *shift = MI_SIZE_BITS - mi_clz(divisor - 1);
+  *magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);
+}
+
+static size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) {
+  mi_assert_internal(n <= UINT32_MAX);
+  const uint64_t hi = ((uint64_t)n * magic) >> 32;
+  return (size_t)((hi + n) >> shift);
+}
+
+bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) {
+  mi_assert(area != NULL);
+  if (area==NULL) return true;
+  mi_assert(page != NULL);
+  if (page == NULL) return true;
+
+  _mi_page_free_collect(page,true);              // collect both thread_delayed and local_free
+  mi_assert_internal(page->local_free == NULL);
+  if (page->used == 0) return true;
+
+  size_t psize;
+  uint8_t* const pstart = mi_page_area(page, &psize);
+  mi_heap_t* const heap = mi_page_heap(page);
+  const size_t bsize    = mi_page_block_size(page);
+  const size_t ubsize   = mi_page_usable_block_size(page); // without padding
+
+  // optimize page with one block
+  if (page->capacity == 1) {
+    mi_assert_internal(page->used == 1 && page->free == NULL);
+    return visitor(heap, area, pstart, ubsize, arg);
+  }
+  mi_assert(bsize <= UINT32_MAX);
+
+  // optimize full pages
+  if (page->used == page->capacity) {
+    uint8_t* block = pstart;
+    for (size_t i = 0; i < page->capacity; i++) {
+      if (!visitor(heap, area, block, ubsize, arg)) return false;
+      block += bsize;
+    }
+    return true;
+  }
+
+  // create a bitmap of free blocks.
+  #define MI_MAX_BLOCKS   (MI_SMALL_PAGE_SIZE / sizeof(void*))
+  uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS];
+  const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS);
+  memset(free_map, 0, bmapsize * sizeof(intptr_t));
+  if (page->capacity % MI_INTPTR_BITS != 0) {
+    // mark left-over bits at the end as free
+    size_t shift   = (page->capacity % MI_INTPTR_BITS);
+    uintptr_t mask = (UINTPTR_MAX << shift);
+    free_map[bmapsize - 1] = mask;
+  }
+
+  // fast repeated division by the block size
+  uint64_t magic;
+  size_t   shift;
+  mi_get_fast_divisor(bsize, &magic, &shift);
+
+  #if MI_DEBUG>1
+  size_t free_count = 0;
+  #endif
+  for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {
+    #if MI_DEBUG>1
+    free_count++;
+    #endif
+    mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize));
+    size_t offset = (uint8_t*)block - pstart;
+    mi_assert_internal(offset % bsize == 0);
+    mi_assert_internal(offset <= UINT32_MAX);
+    size_t blockidx = mi_fast_divide(offset, magic, shift);
+    mi_assert_internal(blockidx == offset / bsize);
+    mi_assert_internal(blockidx < MI_MAX_BLOCKS);
+    size_t bitidx = (blockidx / MI_INTPTR_BITS);
+    size_t bit = blockidx - (bitidx * MI_INTPTR_BITS);
+    free_map[bitidx] |= ((uintptr_t)1 << bit);
+  }
+  mi_assert_internal(page->capacity == (free_count + page->used));
+
+  // walk through all blocks skipping the free ones
+  #if MI_DEBUG>1
+  size_t used_count = 0;
+  #endif
+  uint8_t* block = pstart;
+  for (size_t i = 0; i < bmapsize; i++) {
+    if (free_map[i] == 0) {
+      // every block is in use
+      for (size_t j = 0; j < MI_INTPTR_BITS; j++) {
+        #if MI_DEBUG>1
+        used_count++;
+        #endif
+        if (!visitor(heap, area, block, ubsize, arg)) return false;
+        block += bsize;
+      }
+    }
+    else {
+      // visit the used blocks in the mask
+      uintptr_t m = ~free_map[i];
+      while (m != 0) {
+        #if MI_DEBUG>1
+        used_count++;
+        #endif
+        size_t bitidx = mi_ctz(m);
+        if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false;
+        m &= m - 1;  // clear least significant bit
+      }
+      block += bsize * MI_INTPTR_BITS;
+    }
+  }
+  mi_assert_internal(page->used == used_count);
+  return true;
+}
+
+
+
+// Separate struct to keep `mi_page_t` out of the public interface
+typedef struct mi_theap_area_ex_s {
+  mi_heap_area_t area;
+  mi_page_t* page;
+} mi_theap_area_ex_t;
+
+typedef bool (mi_theap_area_visit_fun)(const mi_theap_t* theap, const mi_theap_area_ex_t* area, void* arg);
+
+static bool mi_theap_visit_areas_page(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) {
+  MI_UNUSED(theap);
+  MI_UNUSED(pq);
+  mi_theap_area_visit_fun* fun = (mi_theap_area_visit_fun*)vfun;
+  mi_theap_area_ex_t xarea;
+  xarea.page = page;
+  _mi_heap_area_init(&xarea.area, page);
+  return fun(theap, &xarea, arg);
+}
+
+// Visit all theap pages as areas
+static bool mi_theap_visit_areas(const mi_theap_t* theap, mi_theap_area_visit_fun* visitor, void* arg) {
+  if (visitor == NULL) return false;
+  return mi_theap_visit_pages((mi_theap_t*)theap, &mi_theap_visit_areas_page, true, (void*)(visitor), arg); // note: function pointer to void* :-{
+}
+
+// Just to pass arguments
+typedef struct mi_visit_blocks_args_s {
+  bool  visit_blocks;
+  mi_block_visit_fun* visitor;
+  void* arg;
+} mi_visit_blocks_args_t;
+
+static bool mi_theap_area_visitor(const mi_theap_t* theap, const mi_theap_area_ex_t* xarea, void* arg) {
+  mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg;
+  if (!args->visitor(_mi_theap_heap(theap), &xarea->area, NULL, xarea->area.block_size, args->arg)) return false;
+  if (args->visit_blocks) {
+    return _mi_theap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg);
+  }
+  else {
+    return true;
+  }
+}
+
+// Visit all blocks in a theap
+bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
+  mi_visit_blocks_args_t args = { visit_blocks, visitor, arg };
+  return mi_theap_visit_areas(theap, &mi_theap_area_visitor, &args);
+}
+
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/threadlocal.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/threadlocal.c
new file mode 100644
index 00000000..eea3329b
--- /dev/null
+++ b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/threadlocal.c
@@ -0,0 +1,236 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2019-2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+/* ----------------------------------------------------------------------------
+Implement dynamic thread local variables (for heap's).
+Unlike most OS native implementations there is no limit on the number
+that can be allocated.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"
+
+/* -----------------------------------------------------------
+  Each thread can have (a dynamically expanding) array of
+  thread-local values. Each slot has a value and a version.
+  The version is used to safely reuse slots.
+----------------------------------------------------------- */
+typedef struct mi_tls_slot_s {
+  size_t  version;
+  void*   value;
+} mi_tls_slot_t;
+
+typedef struct mi_thread_locals_s {
+  size_t        count;
+  mi_tls_slot_t slots[1];
+} mi_thread_locals_t;
+
+static mi_thread_locals_t mi_thread_locals_empty = { 0, {{0,NULL}} };
+
+mi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty;  // always point to a valid `mi_thread_locals_t`
+
+
+/* -----------------------------------------------------------
+  Each key consists of the slot index in the lower bits,
+  and its version it the top bits. When we get a value
+  the version must match or we return NULL. When we set
+  a value, we also set the version of the key.
+----------------------------------------------------------- */
+
+#if MI_SIZE_BITS < 64
+#define MI_TLS_IDX_BITS     (MI_SIZE_BITS/2)      // half for the index, half for the version
+#else
+#define MI_TLS_IDX_BITS     (MI_SIZE_BITS/4)      // 16 bits for the index, 48 bits for the version
+#endif
+#define MI_TLS_IDX_MASK     ((MI_ZU(1)<<MI_TLS_IDX_BITS)-1)
+#define MI_TLS_IDX_MAX      MI_TLS_IDX_MASK
+#define MI_TLS_VERSION_MAX  ((MI_ZU(1)<<(MI_SIZE_BITS - MI_TLS_IDX_BITS))-1)
+
+static size_t mi_key_index( size_t key ) {
+  return (key & MI_TLS_IDX_MASK);
+}
+
+static size_t mi_key_version( size_t key ) {
+  return (key >> MI_TLS_IDX_BITS);
+}
+
+static mi_thread_local_t mi_key_create( size_t index, size_t version ) {
+  mi_assert_internal(version != 0 && version <= MI_TLS_VERSION_MAX);
+  mi_assert_internal(index <= MI_TLS_IDX_MAX);
+  const mi_thread_local_t key = ((version << MI_TLS_IDX_BITS) | index);
+  mi_assert_internal(key != 0);
+  return key;
+}
+
+
+// dynamically reallocate the thread local slots when needed
+static mi_thread_locals_t* mi_thread_locals_expand(size_t least_idx) {
+  mi_thread_locals_t* tls_old = mi_thread_locals;
+  const size_t count_old = tls_old->count;
+  size_t count;
+  if (count_old==0) {
+    tls_old = NULL; // so we allocate fresh from mi_thread_locals_empty
+    count = 16;     // start with 16 slots
+  } 
+  else if (count_old >= 1024) {
+    count = count_old + 1024;  // at some point increase linearly
+  }
+  else {
+    count = 2*count_old;       // and double initially
+  }
+  if (count <= least_idx) {   
+    count = least_idx + 1;
+  }
+  if (count > MI_TLS_IDX_MAX) { return NULL; }  // too large
+  mi_thread_locals_t* tls = (mi_thread_locals_t*)mi_rezalloc(tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t));
+  if mi_unlikely(tls==NULL) return NULL;
+  tls->count = count;
+  mi_thread_locals = tls;
+  return tls;
+}
+
+static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key, void* val ) {
+  if (val==NULL) return true;
+  const size_t idx = mi_key_index(key);  
+  mi_thread_locals_t* tls = mi_thread_locals_expand(idx);
+  if (tls==NULL) return false;
+  mi_assert_internal(tls == mi_thread_locals);
+  mi_assert_internal(idx < tls->count);
+  tls->slots[idx].value = val;
+  tls->slots[idx].version = mi_key_version(key);
+  return true;
+}
+
+// set a tls slot; returns `true` if successful.
+// Can return `false` if we could not reallocate the slots array.
+bool _mi_thread_local_set( mi_thread_local_t key, void* val ) {
+  mi_thread_locals_t* tls = mi_thread_locals;
+  mi_assert_internal(tls!=NULL);
+  mi_assert_internal(key!=0);
+  const size_t idx = mi_key_index(key);
+  if mi_likely(idx < tls->count) {
+    tls->slots[idx].value = val;
+    tls->slots[idx].version = mi_key_version(key);
+    return true;
+  }
+  else {
+    return mi_thread_local_set_expand( key, val );  // tailcall
+  }
+}
+
+// get a tls slot value
+void* _mi_thread_local_get( mi_thread_local_t key ) {
+  const mi_thread_locals_t* const tls = mi_thread_locals;
+  mi_assert_internal(tls!=NULL);
+  mi_assert_internal(key!=0);
+  const size_t idx = mi_key_index(key);
+  if mi_likely(idx < tls->count && mi_key_version(key) == tls->slots[idx].version) {
+    return tls->slots[idx].value;
+  }
+  else {
+    return NULL;  
+  }
+}
+
+void _mi_thread_locals_thread_done(void) {
+  mi_thread_locals_t* const tls = mi_thread_locals;
+  if (tls!=NULL && tls->count > 0) {
+    mi_free(tls);
+    mi_thread_locals = &mi_thread_locals_empty;
+  }
+}
+
+/* -----------------------------------------------------------
+Create and free fresh TLS key's
+----------------------------------------------------------- */
+#include "bitmap.h"
+
+static mi_lock_t    mi_thread_locals_lock;    // we need a lock in order to re-allocate the slot bits
+static mi_bitmap_t* mi_thread_locals_free;    // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use)
+static size_t       mi_thread_locals_version; // version to be able to reuse slots safely
+
+void _mi_thread_locals_init(void) {
+  mi_lock_init(&mi_thread_locals_lock);
+}
+
+void _mi_thread_locals_done(void) {
+  mi_lock(&mi_thread_locals_lock) {
+    mi_bitmap_t* const slots = mi_thread_locals_free;
+    mi_free(slots);
+  }
+  mi_lock_done(&mi_thread_locals_lock);
+}
+
+// strange signature but allows us to reuse the arena code for claiming free pages
+static bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, bool* keep_set) {
+  MI_UNUSED(_slice_index); MI_UNUSED(_arena);
+  *keep_set = false;
+  return true;
+}
+
+// When we claim a free slot, we increase the global version counter 
+// (so if we reuse a slot it will be returning NULL initially when a thread tries to get it)
+static mi_thread_local_t mi_thread_local_claim(void) {
+  size_t idx = 0;
+  if (mi_thread_locals_free != NULL && mi_bitmap_try_find_and_claim(mi_thread_locals_free,0,&idx,&mi_thread_local_claim_fun,NULL)) {
+    mi_thread_locals_version++;
+    if (mi_thread_locals_version >= MI_TLS_VERSION_MAX) { mi_thread_locals_version = 1; }  /* wrap around the version */
+    return mi_key_create( idx, mi_thread_locals_version);
+  }
+  else {
+    return 0;
+  }
+}
+
+static bool mi_thread_local_create_expand(void) {
+  mi_bitmap_t* const slots = mi_thread_locals_free;
+  // 1024 bits at a time
+  const size_t oldcount = (slots==NULL ? 0 : mi_bitmap_max_bits(slots));
+  const size_t newcount = 1024 + oldcount;
+  if (newcount > MI_TLS_IDX_MAX) { return false; }
+  const size_t newsize = mi_bitmap_size( newcount, NULL );
+  mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE);
+  if (newslots==NULL) { return false; }
+  if (slots!=NULL) {
+    // copy over the previous bitmap
+    _mi_memcpy_aligned(newslots, slots, mi_bitmap_size(oldcount, NULL)); 
+    mi_free(slots);
+  }
+  mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */);
+  mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount);  /* set the new expanded slots as available */
+  mi_thread_locals_free = newslots;
+  return true;
+}
+
+
+// create a fresh key
+mi_thread_local_t _mi_thread_local_create(void) {
+  mi_thread_local_t key = 0;
+  mi_lock(&mi_thread_locals_lock) {
+    key = mi_thread_local_claim();
+    if (key==0) {
+      if (mi_thread_local_create_expand()) {
+        key = mi_thread_local_claim();
+      }
+    }
+  }
+  return key;
+}
+
+// free a key
+void _mi_thread_local_free(mi_thread_local_t key) {
+  if (key==0) return;
+  const size_t idx = mi_key_index(key);
+  mi_lock(&mi_thread_locals_lock) {
+    mi_bitmap_t* const slots = mi_thread_locals_free;
+    if (slots!=NULL && idx < mi_bitmap_max_bits(slots)) {
+      mi_bitmap_set(slots,idx);
+    }
+  }
+}
+
diff --git a/vendor/libmimalloc-sys/src/extended.rs b/vendor/libmimalloc-sys/src/extended.rs
new file mode 100644
index 00000000..90a767fc
--- /dev/null
+++ b/vendor/libmimalloc-sys/src/extended.rs
@@ -0,0 +1,1126 @@
+#![allow(nonstandard_style)]
+
+use core::ffi::c_void;
+
+use cty::{c_char, c_int, c_long, c_ulonglong};
+
+/// The maximum number of bytes which may be used as an argument to a function
+/// in the `_small` family ([`mi_malloc_small`], [`mi_zalloc_small`], etc).
+pub const MI_SMALL_SIZE_MAX: usize = 128 * core::mem::size_of::<*mut c_void>();
+
+extern "C" {
+    /// Allocate `count` items of `size` length each.
+    ///
+    /// Returns `null` if `count * size` overflows or on out-of-memory.
+    ///
+    /// All items are initialized to zero.
+    pub fn mi_calloc(count: usize, size: usize) -> *mut c_void;
+
+    /// Allocate `count` items of `size` length each.
+    ///
+    /// Returns `null` if `count * size` overflows or on out-of-memory,
+    /// otherwise returns the same as [`mi_malloc(count *
+    /// size)`](crate::mi_malloc).
+    /// Equivalent to [`mi_calloc`], but returns uninitialized (and not zeroed)
+    /// bytes.
+    pub fn mi_mallocn(count: usize, size: usize) -> *mut c_void;
+
+    /// Re-allocate memory to `count` elements of `size` bytes.
+    ///
+    /// The realloc equivalent of the [`mi_mallocn`] interface. Returns `null`
+    /// if `count * size` overflows or on out-of-memory, otherwise returns the
+    /// same as [`mi_realloc(p, count * size)`](crate::mi_realloc).
+    pub fn mi_reallocn(p: *mut c_void, count: usize, size: usize) -> *mut c_void;
+
+    /// Try to re-allocate memory to `newsize` bytes _in place_.
+    ///
+    /// Returns null on out-of-memory or if the memory could not be expanded in
+    /// place. On success, returns the same pointer as `p`.
+    ///
+    /// If `newsize` is larger than the original `size` allocated for `p`, the
+    /// bytes after `size` are uninitialized.
+    ///
+    /// If null is returned, the original pointer is not freed.
+    ///
+    /// Note: Conceptually, this is a realloc-like which returns null if it
+    /// would be forced to reallocate memory and copy. In practice it's
+    /// equivalent testing against [`mi_usable_size`](crate::mi_usable_size).
+    pub fn mi_expand(p: *mut c_void, newsize: usize) -> *mut c_void;
+
+    /// Re-allocate memory to `newsize` bytes.
+    ///
+    /// This differs from [`mi_realloc`](crate::mi_realloc) in that on failure,
+    /// `p` is freed.
+    pub fn mi_reallocf(p: *mut c_void, newsize: usize) -> *mut c_void;
+
+    /// Allocate and duplicate a nul-terminated C string.
+    ///
+    /// This can be useful for Rust code when interacting with the FFI.
+    pub fn mi_strdup(s: *const c_char) -> *mut c_char;
+
+    /// Allocate and duplicate a nul-terminated C string, up to `n` bytes.
+    ///
+    /// This can be useful for Rust code when interacting with the FFI.
+    pub fn mi_strndup(s: *const c_char, n: usize) -> *mut c_char;
+
+    /// Resolve a file path name, producing a `C` string which can be passed to
+    /// [`mi_free`](crate::mi_free).
+    ///
+    /// `resolved_name` should be null, but can also point to a buffer of at
+    /// least `PATH_MAX` bytes.
+    ///
+    /// If successful, returns a pointer to the resolved absolute file name, or
+    /// `null` on failure (with `errno` set to the error code).
+    ///
+    /// If `resolved_name` was `null`, the returned result should be freed with
+    /// [`mi_free`](crate::mi_free).
+    ///
+    /// This can rarely be useful in FFI code, but is mostly included for
+    /// completeness.
+    pub fn mi_realpath(fname: *const c_char, resolved_name: *mut c_char) -> *mut c_char;
+
+    /// Allocate `size * count` bytes aligned by `alignment`.
+    ///
+    /// Return pointer to the allocated memory or null if out of memory or if
+    /// `size * count` overflows.
+    ///
+    /// Returns a unique pointer if called with `size * count` 0.
+    pub fn mi_calloc_aligned(count: usize, size: usize, alignment: usize) -> *mut c_void;
+
+    /// Allocate `size` bytes aligned by `alignment` at a specified `offset`.
+    ///
+    /// Note that the resulting pointer itself is not aligned by the alignment,
+    /// but after `offset` bytes it will be. This can be useful for allocating
+    /// data with an inline header, where the data has a specific alignment
+    /// requirement.
+    ///
+    /// Specifically, if `p` is the returned pointer `p.add(offset)` is aligned
+    /// to `alignment`.
+    pub fn mi_malloc_aligned_at(size: usize, alignment: usize, offset: usize) -> *mut c_void;
+
+    /// Allocate `size` bytes aligned by `alignment` at a specified `offset`,
+    /// zero-initialized.
+    ///
+    /// This is a [`mi_zalloc`](crate::mi_zalloc) equivalent of [`mi_malloc_aligned_at`].
+    pub fn mi_zalloc_aligned_at(size: usize, alignment: usize, offset: usize) -> *mut c_void;
+
+    /// Allocate `size` of bytes aligned by `alignment` and place the address of the
+    /// allocated memory to `ptr`.
+    ///
+    /// Returns zero on success, invalid argument for invalid alignment, or out-of-memory.
+    pub fn mi_posix_memalign(ptr: *mut *mut c_void, alignment: usize, size: usize) -> c_int;
+
+    /// Allocate `size` bytes aligned by `alignment` with alignment as the first
+    /// parameter.
+    ///
+    /// Return pointer to the allocated memory or null if out of memory.
+    pub fn mi_aligned_alloc(alignment: usize, size: usize) -> *mut c_void;
+
+    /// Allocate `size * count` bytes aligned by `alignment` at a specified
+    /// `offset`, zero-initialized.
+    ///
+    /// This is a [`calloc`](crate::mi_calloc) equivalent of [`mi_malloc_aligned_at`].
+    pub fn mi_calloc_aligned_at(
+        count: usize,
+        size: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Re-allocate memory to `newsize` bytes aligned by `alignment` at a
+    /// specified `offset`.
+    ///
+    /// This is a [`realloc`](crate::mi_realloc) equivalent of [`mi_malloc_aligned_at`].
+    pub fn mi_realloc_aligned_at(
+        p: *mut c_void,
+        newsize: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Zero initialized [re-allocation](crate::mi_realloc).
+    ///
+    /// In general, only valid on memory originally allocated by zero
+    /// initialization: [`mi_calloc`](crate::mi_calloc),
+    /// [`mi_zalloc`](crate::mi_zalloc),
+    /// [`mi_zalloc_aligned`](crate::mi_zalloc_aligned), ...
+    pub fn mi_rezalloc(p: *mut c_void, newsize: usize) -> *mut c_void;
+
+    /// Zero initialized [re-allocation](crate::mi_realloc), following `calloc`
+    /// paramater conventions.
+    ///
+    /// In general, only valid on memory originally allocated by zero
+    /// initialization: [`mi_calloc`](crate::mi_calloc),
+    /// [`mi_zalloc`](crate::mi_zalloc),
+    /// [`mi_zalloc_aligned`](crate::mi_zalloc_aligned), ...
+    pub fn mi_recalloc(p: *mut c_void, newcount: usize, size: usize) -> *mut c_void;
+
+    /// Aligned version of [`mi_rezalloc`].
+    pub fn mi_rezalloc_aligned(p: *mut c_void, newsize: usize, alignment: usize) -> *mut c_void;
+
+    /// Offset-aligned version of [`mi_rezalloc`].
+    pub fn mi_rezalloc_aligned_at(
+        p: *mut c_void,
+        newsize: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Aligned version of [`mi_recalloc`].
+    pub fn mi_recalloc_aligned(
+        p: *mut c_void,
+        newcount: usize,
+        size: usize,
+        alignment: usize,
+    ) -> *mut c_void;
+
+    /// Offset-aligned version of [`mi_recalloc`].
+    pub fn mi_recalloc_aligned_at(
+        p: *mut c_void,
+        newcount: usize,
+        size: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Allocate an object of no more than [`MI_SMALL_SIZE_MAX`] bytes.
+    ///
+    /// Does not check that `size` is indeed small.
+    ///
+    /// Note: Currently [`mi_malloc`](crate::mi_malloc) checks if `size` is
+    /// small and calls this if
+    /// so at runtime, so its' only worth using if you know for certain.
+    pub fn mi_malloc_small(size: usize) -> *mut c_void;
+
+    /// Allocate an zero-initialized object of no more than
+    /// [`MI_SMALL_SIZE_MAX`] bytes.
+    ///
+    /// Does not check that `size` is indeed small.
+    ///
+    /// Note: Currently [`mi_zalloc`](crate::mi_zalloc) checks if `size` is
+    /// small and calls this if so at runtime, so its' only worth using if you
+    /// know for certain.
+    pub fn mi_zalloc_small(size: usize) -> *mut c_void;
+
+    /// Free a small object. Only use to free objects from [`mi_malloc_small`]
+    /// or [`mi_zalloc_small`]. Potentially a tiny bit faster than [`mi_free`](crate::mi_free).
+    pub fn mi_free_small(p: *mut c_void);
+
+    /// Return the available bytes in a memory block.
+    ///
+    /// The returned size can be used to call `mi_expand` successfully.
+    pub fn mi_usable_size(p: *const c_void) -> usize;
+
+    /// Return the used allocation size.
+    ///
+    /// Returns the size `n` that will be allocated, where `n >= size`.
+    ///
+    /// Generally, `mi_usable_size(mi_malloc(size)) == mi_good_size(size)`. This
+    /// can be used to reduce internal wasted space when allocating buffers for
+    /// example.
+    ///
+    /// See [`mi_usable_size`](crate::mi_usable_size).
+    pub fn mi_good_size(size: usize) -> usize;
+
+    /// Eagerly free memory.
+    ///
+    /// If `force` is true, aggressively return memory to the OS (can be
+    /// expensive!)
+    ///
+    /// Regular code should not have to call this function. It can be beneficial
+    /// in very narrow circumstances; in particular, when a long running thread
+    /// allocates a lot of blocks that are freed by other threads it may improve
+    /// resource usage by calling this every once in a while.
+    pub fn mi_collect(force: bool);
+
+    /// Checked free: If `p` came from mimalloc's heap (as decided by
+    /// [`mi_is_in_heap_region`]), this is [`mi_free(p)`](crate::mi_free), but
+    /// otherwise it is a no-op.
+    pub fn mi_cfree(p: *mut c_void);
+
+    /// Returns true if this is a pointer into a memory region that has been
+    /// reserved by the mimalloc heap.
+    ///
+    /// This function is described by the mimalloc documentation as "relatively
+    /// fast".
+    ///
+    /// See also [`mi_heap_check_owned`], which is (much) slower and slightly
+    /// more precise, but only concerns a single `mi_heap`.
+    pub fn mi_is_in_heap_region(p: *const c_void) -> bool;
+
+    /// Layout-aware deallocation: Like [`mi_free`](crate::mi_free), but accepts
+    /// the size and alignment as well.
+    ///
+    /// Note: unlike some allocators that require this information for
+    /// performance, mimalloc doesn't need it (as of the current version,
+    /// v2.0.0), and so it currently implements this as a (debug) assertion that
+    /// verifies that `p` is actually aligned to `alignment` and is usable for
+    /// at least `size` bytes, before delegating to `mi_free`.
+    ///
+    /// However, currently there's no way to have this crate enable mimalloc's
+    /// debug assertions, so these checks aren't particularly useful.
+    ///
+    /// Note: It's legal to pass null to this function, and you are not required
+    /// to use this to deallocate memory from an aligned allocation function.
+    pub fn mi_free_size_aligned(p: *mut c_void, size: usize, alignment: usize);
+
+    /// Size-aware deallocation: Like [`mi_free`](crate::mi_free), but accepts
+    /// the size and alignment as well.
+    ///
+    /// Note: unlike some allocators that require this information for
+    /// performance, mimalloc doesn't need it (as of the current version,
+    /// v2.0.0), and so it currently implements this as a (debug) assertion that
+    /// verifies that `p` is actually aligned to `alignment` and is usable for
+    /// at least `size` bytes, before delegating to `mi_free`.
+    ///
+    /// However, currently there's no way to have this crate enable mimalloc's
+    /// debug assertions, so these checks aren't particularly useful.
+    ///
+    /// Note: It's legal to pass null to this function.
+    pub fn mi_free_size(p: *mut c_void, size: usize);
+
+    /// Alignment-aware deallocation: Like [`mi_free`](crate::mi_free), but
+    /// accepts the size and alignment as well.
+    ///
+    /// Note: unlike some allocators that require this information for
+    /// performance, mimalloc doesn't need it (as of the current version,
+    /// v2.0.0), and so it currently implements this as a (debug) assertion that
+    /// verifies that `p` is actually aligned to `alignment` and is usable for
+    /// at least `size` bytes, before delegating to `mi_free`.
+    ///
+    /// However, currently there's no way to have this crate enable mimalloc's
+    /// debug assertions, so these checks aren't particularly useful.
+    ///
+    /// Note: It's legal to pass null to this function.
+    pub fn mi_free_aligned(p: *mut c_void, alignment: usize);
+
+    /// Print the main statistics.
+    ///
+    /// Ignores the passed in argument, and outputs to the registered output
+    /// function or stderr by default.
+    ///
+    /// Most detailed when using a debug build.
+    pub fn mi_stats_print(_: *mut c_void);
+
+    /// Print the main statistics.
+    ///
+    /// Pass `None` for `out` to use the default. If `out` is provided, `arc` is
+    /// passed as it's second parameter.
+    ///
+    /// Most detailed when using a debug build.
+    pub fn mi_stats_print_out(out: mi_output_fun, arg: *mut c_void);
+
+    /// Reset statistics.
+    ///
+    /// Note: This function is thread safe.
+    pub fn mi_stats_reset();
+
+    /// Merge thread local statistics with the main statistics and reset.
+    ///
+    /// Note: This function is thread safe.
+    #[cfg(feature = "v2")]
+    pub fn mi_stats_merge();
+
+    /// Return the mimalloc version number.
+    ///
+    /// For example version 1.6.3 would return the number `163`.
+    pub fn mi_version() -> c_int;
+
+    /// Initialize mimalloc on a thread.
+    ///
+    /// Should not be used as on most systems (pthreads, windows) this is done
+    /// automatically.
+    pub fn mi_thread_init();
+
+    /// Initialize the process.
+    ///
+    /// Should not be used on most systems, as it's called by thread_init or the
+    /// process loader.
+    pub fn mi_process_init();
+
+    /// Return process information (time and memory usage). All parameters are
+    /// optional (nullable) out-params:
+    ///
+    /// | Parameter        | Description |
+    /// | :-               | :- |
+    /// | `elapsed_msecs`  | Elapsed wall-clock time of the process in milli-seconds. |
+    /// | `user_msecs`     | User time in milli-seconds (as the sum over all threads). |
+    /// | `system_msecs`   | System time in milli-seconds. |
+    /// | `current_rss`    | Current working set size (touched pages). |
+    /// | `peak_rss`       | Peak working set size (touched pages). |
+    /// | `current_commit` | Current committed memory (backed by the page file). |
+    /// | `peak_commit`    | Peak committed memory (backed by the page file). |
+    /// | `page_faults`    | Count of hard page faults. |
+    ///
+    /// The `current_rss` is precise on Windows and MacOSX; other systems
+    /// estimate this using `current_commit`. The `commit` is precise on Windows
+    /// but estimated on other systems as the amount of read/write accessible
+    /// memory reserved by mimalloc.
+    pub fn mi_process_info(
+        elapsed_msecs: *mut usize,
+        user_msecs: *mut usize,
+        system_msecs: *mut usize,
+        current_rss: *mut usize,
+        peak_rss: *mut usize,
+        current_commit: *mut usize,
+        peak_commit: *mut usize,
+        page_faults: *mut usize,
+    );
+
+    /// Uninitialize mimalloc on a thread.
+    ///
+    /// Should not be used as on most systems (pthreads, windows) this is done
+    /// automatically. Ensures that any memory that is not freed yet (but will
+    /// be freed by other threads in the future) is properly handled.
+    ///
+    /// Note: This function is thread safe.
+    pub fn mi_thread_done();
+
+    /// Print out heap statistics for this thread.
+    ///
+    /// Pass `None` for `out` to use the default. If `out` is provided, `arc` is
+    /// passed as it's second parameter
+    ///
+    /// Most detailed when using a debug build.
+    ///
+    /// Note: This function is thread safe.
+    pub fn mi_thread_stats_print_out(out: mi_output_fun, arg: *mut c_void);
+
+    /// Register an output function.
+    ///
+    /// - `out` The output function, use `None` to output to stderr.
+    /// - `arg` Argument that will be passed on to the output function.
+    ///
+    /// The `out` function is called to output any information from mimalloc,
+    /// like verbose or warning messages.
+    ///
+    /// Note: This function is thread safe.
+    pub fn mi_register_output(out: mi_output_fun, arg: *mut c_void);
+
+    /// Register a deferred free function.
+    ///
+    /// - `deferred_free` Address of a deferred free-ing function or `None` to
+    ///   unregister.
+    /// - `arg` Argument that will be passed on to the deferred free function.
+    ///
+    /// Some runtime systems use deferred free-ing, for example when using
+    /// reference counting to limit the worst case free time.
+    ///
+    /// Such systems can register (re-entrant) deferred free function to free
+    /// more memory on demand.
+    ///
+    /// - When the `force` parameter is `true` all possible memory should be
+    ///   freed.
+    ///
+    /// - The per-thread `heartbeat` parameter is monotonically increasing and
+    ///   guaranteed to be deterministic if the program allocates
+    ///   deterministically.
+    ///
+    /// - The `deferred_free` function is guaranteed to be called
+    ///   deterministically after some number of allocations (regardless of
+    ///   freeing or available free memory).
+    ///
+    /// At most one `deferred_free` function can be active.
+    ///
+    /// Note: This function is thread safe.
+    pub fn mi_register_deferred_free(out: mi_deferred_free_fun, arg: *mut c_void);
+
+    /// Register an error callback function.
+    ///
+    /// The `errfun` function is called on an error in mimalloc after emitting
+    /// an error message (through the output function).
+    ///
+    /// It as always legal to just return from the `errfun` function in which
+    /// case allocation functions generally return null or ignore the condition.
+    ///
+    /// The default function only calls abort() when compiled in secure mode
+    /// with an `EFAULT` error. The possible error codes are:
+    ///
+    /// - `EAGAIN` (11): Double free was detected (only in debug and secure
+    ///   mode).
+    /// - `EFAULT` (14): Corrupted free list or meta-data was detected (only in
+    ///   debug and secure mode).
+    /// - `ENOMEM` (12): Not enough memory available to satisfy the request.
+    /// - `EOVERFLOW` (75): Too large a request, for example in `mi_calloc`, the
+    ///   `count` and `size` parameters are too large.
+    /// - `EINVAL` (22): Trying to free or re-allocate an invalid pointer.
+    ///
+    /// Note: This function is thread safe.
+    pub fn mi_register_error(out: mi_error_fun, arg: *mut c_void);
+
+    /// Get the statistics for the current subprocess aggregated over all its heaps as JSON.
+    ///
+    /// Returns pointer to the buffer or NULL on failure. Use mi_free() to free the buffer if the buf parameter was NULL.
+    #[cfg(not(feature = "v2"))]
+    pub fn mi_stats_get_json(buf_size: usize, buf: *mut c_char) -> *mut c_char;
+}
+
+/// An output callback. Must be thread-safe.
+///
+/// See [`mi_stats_print_out`], [`mi_thread_stats_print_out`], [`mi_register_output`]
+pub type mi_output_fun = Option<unsafe extern "C" fn(msg: *const c_char, arg: *mut c_void)>;
+
+/// Type of deferred free functions. Must be thread-safe.
+///
+/// - `force`: If true, all outstanding items should be freed.
+/// - `heartbeat` A monotonically increasing count.
+/// - `arg` Argument that was passed at registration to hold extra state.
+///
+/// See [`mi_register_deferred_free`]
+pub type mi_deferred_free_fun =
+    Option<unsafe extern "C" fn(force: bool, heartbeat: c_ulonglong, arg: *mut c_void)>;
+
+/// Type of error callback functions. Must be thread-safe.
+///
+/// - `err`: Error code (see [`mi_register_error`] for a list).
+/// - `arg`: Argument that was passed at registration to hold extra state.
+///
+/// See [`mi_register_error`]
+pub type mi_error_fun = Option<unsafe extern "C" fn(code: c_int, arg: *mut c_void)>;
+
+/// Runtime options. All options are false by default.
+pub type mi_option_t = c_int;
+
+#[cfg(all(feature = "arena", feature = "v2"))]
+/// Arena Id
+pub type mi_arena_id_t = c_int;
+
+#[cfg(all(feature = "arena", not(feature = "v2")))]
+/// Arena Id
+pub type mi_arena_id_t = *mut c_void;
+
+// Note: mimalloc doc website seems to have the order of show_stats and
+// show_errors reversed as of 1.6.3, however what I have here is correct:
+// https://github.com/microsoft/mimalloc/issues/266#issuecomment-653822341
+
+/// Print error messages to `stderr`.
+pub const mi_option_show_errors: mi_option_t = 0;
+
+/// Print statistics to `stderr` when the program is done.
+pub const mi_option_show_stats: mi_option_t = 1;
+
+/// Print verbose messages to `stderr`.
+pub const mi_option_verbose: mi_option_t = 2;
+
+/// ### The following options are experimental
+///
+/// Option (experimental) Use large OS pages (2MiB in size) if possible.
+///
+/// Use large OS pages (2MiB) when available; for some workloads this can
+/// significantly improve performance. Use mi_option_verbose to check if
+/// the large OS pages are enabled -- usually one needs to explicitly allow
+/// large OS pages (as on Windows and Linux). However, sometimes the OS is
+/// very slow to reserve contiguous physical memory for large OS pages so
+/// use with care on systems that can have fragmented memory (for that
+/// reason, we generally recommend to use mi_option_reserve_huge_os_pages
+/// instead whenever possible).
+pub const mi_option_large_os_pages: mi_option_t = 6;
+
+/// Option (experimental) The number of huge OS pages (1GiB in size) to reserve at the start of the program.
+///
+/// This reserves the huge pages at startup and sometimes this can give a large (latency) performance
+/// improvement on big workloads. Usually it is better to not use MIMALLOC_LARGE_OS_PAGES in
+/// combination with this setting. Just like large OS pages, use with care as reserving contiguous
+/// physical memory can take a long time when memory is fragmented (but reserving the huge pages is
+/// done at startup only once). Note that we usually need to explicitly enable huge OS pages (as on
+/// Windows and Linux)). With huge OS pages, it may be beneficial to set the setting
+/// mi_option_eager_commit_delay=N (N is 1 by default) to delay the initial N segments (of 4MiB) of
+/// a thread to not allocate in the huge OS pages; this prevents threads that are short lived and
+/// allocate just a little to take up space in the huge OS page area (which cannot be reset).
+pub const mi_option_reserve_huge_os_pages: mi_option_t = 7;
+
+/// Option (experimental) Reserve huge OS pages at node N.
+///
+/// The huge pages are usually allocated evenly among NUMA nodes.
+/// You can use mi_option_reserve_huge_os_pages_at=N where `N` is the numa node (starting at 0) to allocate all
+/// the huge pages at a specific numa node instead.
+pub const mi_option_reserve_huge_os_pages_at: mi_option_t = 8;
+
+/// Option (experimental) Reserve specified amount of OS memory at startup, e.g. "1g" or "512m".
+pub const mi_option_reserve_os_memory: mi_option_t = 9;
+
+#[cfg(feature = "v2")]
+/// Option (experimental) the first N segments per thread are not eagerly committed (=1).
+pub const mi_option_eager_commit_delay: mi_option_t = 14;
+
+/// Option (experimental) Pretend there are at most N NUMA nodes; Use 0 to use the actual detected NUMA nodes at runtime.
+pub const mi_option_use_numa_nodes: mi_option_t = 16;
+
+/// Option (experimental) If set to 1, do not use OS memory for allocation (but only pre-reserved arenas)
+pub const mi_option_limit_os_alloc: mi_option_t = 17;
+
+/// Option (experimental) OS tag to assign to mimalloc'd memory
+pub const mi_option_os_tag: mi_option_t = 18;
+
+/// Option (experimental)
+pub const mi_option_max_errors: mi_option_t = 19;
+
+/// Option (experimental)
+pub const mi_option_max_warnings: mi_option_t = 20;
+
+#[cfg(feature = "v2")]
+/// Option (experimental)
+pub const mi_option_max_segment_reclaim: mi_option_t = 21;
+
+/// Last option.
+#[cfg(feature = "v2")]
+pub const _mi_option_last: mi_option_t = 38;
+#[cfg(not(feature = "v2"))]
+pub const _mi_option_last: mi_option_t = 47;
+
+extern "C" {
+    // Note: mi_option_{enable,disable} aren't exposed because they're redundant
+    // and because of https://github.com/microsoft/mimalloc/issues/266.
+
+    /// Returns true if the provided option is enabled.
+    ///
+    /// Note: this function is not thread safe.
+    pub fn mi_option_is_enabled(option: mi_option_t) -> bool;
+
+    /// Enable or disable the given option.
+    ///
+    /// Note: this function is not thread safe.
+    pub fn mi_option_set_enabled(option: mi_option_t, enable: bool);
+
+    /// If the given option has not yet been initialized with [`mi_option_set`]
+    /// or [`mi_option_set_enabled`], enables or disables the option. If it has,
+    /// this function does nothing.
+    ///
+    /// Note: this function is not thread safe.
+    pub fn mi_option_set_enabled_default(option: mi_option_t, enable: bool);
+
+    /// Returns the value of the provided option.
+    ///
+    /// The value of boolean options is 1 or 0, however experimental options
+    /// exist which take a numeric value, which is the intended use of this
+    /// function.
+    ///
+    /// These options are not exposed as constants for stability reasons,
+    /// however you can still use them as arguments to this and other
+    /// `mi_option_` functions if needed, see the mimalloc documentation for
+    /// details: https://microsoft.github.io/mimalloc/group__options.html
+    ///
+    /// Note: this function is not thread safe.
+    pub fn mi_option_get(option: mi_option_t) -> c_long;
+
+    /// Set the option to the given value.
+    ///
+    /// The value of boolean options is 1 or 0, however experimental options
+    /// exist which take a numeric value, which is the intended use of this
+    /// function.
+    ///
+    /// These options are not exposed as constants for stability reasons,
+    /// however you can still use them as arguments to this and other
+    /// `mi_option_` functions if needed,
+    ///
+    /// Note: this function is not thread safe.
+    pub fn mi_option_set(option: mi_option_t, value: c_long);
+
+    /// If the given option has not yet been initialized with [`mi_option_set`]
+    /// or [`mi_option_set_enabled`], sets the option to the given value. If it
+    /// has, this function does nothing.
+    ///
+    /// The value of boolean options is 1 or 0, however experimental options
+    /// exist which take a numeric value, which is the intended use of this
+    /// function.
+    ///
+    /// These options are not exposed as constants for stability reasons,
+    /// however you can still use them as arguments to this and other
+    /// `mi_option_` functions if needed.
+    ///
+    /// Note: this function is not thread safe.
+    pub fn mi_option_set_default(option: mi_option_t, value: c_long);
+}
+
+/// First-class heaps that can be destroyed in one go.
+///
+/// Note: The pointers allocated out of a heap can be be freed using
+/// [`mi_free`](crate::mi_free) -- there is no `mi_heap_free`.
+///
+/// # Example
+///
+/// ```
+/// use libmimalloc_sys as mi;
+/// unsafe {
+///     let h = mi::mi_heap_new();
+///     assert!(!h.is_null());
+///     let p = mi::mi_heap_malloc(h, 50);
+///     assert!(!p.is_null());
+///
+///     // use p...
+///     mi::mi_free(p);
+///
+///     // Clean up the heap. Note that pointers allocated from `h`
+///     // are *not* invalided by `mi_heap_delete`. You would have
+///     // to use (the very dangerous) `mi_heap_destroy` for that
+///     // behavior
+///     mi::mi_heap_delete(h);
+/// }
+/// ```
+pub enum mi_heap_t {}
+
+#[cfg(not(feature = "v2"))]
+pub enum mi_theap_t {}
+
+/// An area of heap space contains blocks of a single size.
+///
+/// The bytes in freed blocks are `committed - used`.
+#[repr(C)]
+#[derive(Debug, Clone, Copy)]
+pub struct mi_heap_area_t {
+    /// Start of the area containing heap blocks.
+    pub blocks: *mut c_void,
+    /// Bytes reserved for this area.
+    pub reserved: usize,
+    /// Current committed bytes of this area.
+    pub committed: usize,
+    /// Bytes in use by allocated blocks.
+    pub used: usize,
+    /// Size in bytes of one block.
+    pub block_size: usize,
+    /// Size in bytes of a full block including padding and metadata.
+    pub full_block_size: usize,
+    /// Heap tag associated with this area (not available in v3)
+    #[cfg(feature = "v2")]
+    pub heap_tag: i32,
+    /// Reserved / internal (replaces `heap_tag` in v3)
+    #[cfg(not(feature = "v2"))]
+    pub reserved1: *mut c_void,
+}
+
+/// Visitor function passed to [`mi_heap_visit_blocks`]
+///
+/// Should return `true` to continue, and `false` to stop visiting (i.e. break)
+///
+/// This function is always first called for every `area` with `block` as a null
+/// pointer. If `visit_all_blocks` was `true`, the function is then called for
+/// every allocated block in that area.
+pub type mi_block_visit_fun = Option<
+    unsafe extern "C" fn(
+        heap: *const mi_heap_t,
+        area: *const mi_heap_area_t,
+        block: *mut c_void,
+        block_size: usize,
+        arg: *mut c_void,
+    ) -> bool,
+>;
+
+extern "C" {
+    /// Create a new heap that can be used for allocation.
+    pub fn mi_heap_new() -> *mut mi_heap_t;
+
+    /// Delete a previously allocated heap.
+    ///
+    /// This will release resources and migrate any still allocated blocks in
+    /// this heap (efficienty) to the default heap.
+    ///
+    /// If `heap` is the default heap, the default heap is set to the backing
+    /// heap.
+    pub fn mi_heap_delete(heap: *mut mi_heap_t);
+
+    /// Destroy a heap, freeing all its still allocated blocks.
+    ///
+    /// Use with care as this will free all blocks still allocated in the heap.
+    /// However, this can be a very efficient way to free all heap memory in one
+    /// go.
+    ///
+    /// If `heap` is the default heap, the default heap is set to the backing
+    /// heap.
+    pub fn mi_heap_destroy(heap: *mut mi_heap_t);
+
+    /// Set the default heap to use for [`mi_malloc`](crate::mi_malloc) et al.
+    ///
+    /// Returns the previous default heap.
+    #[cfg(feature = "v2")]
+    pub fn mi_heap_set_default(heap: *mut mi_heap_t) -> *mut mi_heap_t;
+
+    /// Get the default heap that is used for [`mi_malloc`](crate::mi_malloc) et al.
+    #[cfg(feature = "v2")]
+    pub fn mi_heap_get_default() -> *mut mi_heap_t;
+
+    /// Get the backing heap.
+    ///
+    /// The _backing_ heap is the initial default heap for a thread and always
+    /// available for allocations. It cannot be destroyed or deleted except by
+    /// exiting the thread.
+    #[cfg(feature = "v2")]
+    pub fn mi_heap_get_backing() -> *mut mi_heap_t;
+
+    /// Release outstanding resources in a specific heap.
+    ///
+    /// See also [`mi_collect`].
+    pub fn mi_heap_collect(heap: *mut mi_heap_t, force: bool);
+
+    /// Equivalent to [`mi_malloc`](crate::mi_malloc), but allocates out of the
+    /// specific heap instead of the default.
+    pub fn mi_heap_malloc(heap: *mut mi_heap_t, size: usize) -> *mut c_void;
+
+    /// Equivalent to [`mi_zalloc`](crate::mi_zalloc), but allocates out of the
+    /// specific heap instead of the default.
+    pub fn mi_heap_zalloc(heap: *mut mi_heap_t, size: usize) -> *mut c_void;
+
+    /// Equivalent to [`mi_calloc`], but allocates out of the specific heap
+    /// instead of the default.
+    pub fn mi_heap_calloc(heap: *mut mi_heap_t, count: usize, size: usize) -> *mut c_void;
+
+    /// Equivalent to [`mi_mallocn`], but allocates out of the specific heap
+    /// instead of the default.
+    pub fn mi_heap_mallocn(heap: *mut mi_heap_t, count: usize, size: usize) -> *mut c_void;
+
+    /// Equivalent to [`mi_malloc_small`], but allocates out of the specific
+    /// heap instead of the default.
+    ///
+    /// `size` must be smaller or equal to [`MI_SMALL_SIZE_MAX`].
+    pub fn mi_heap_malloc_small(heap: *mut mi_heap_t, size: usize) -> *mut c_void;
+
+    /// Equivalent to [`mi_zalloc_small`], but allocates out of the specific
+    /// heap instead of the default.
+    ///
+    /// `size` must be smaller or equal to [`MI_SMALL_SIZE_MAX`].
+    pub fn mi_heap_zalloc_small(heap: *mut mi_heap_t, size: usize) -> *mut c_void;
+
+    /// Equivalent to [`mi_realloc`](crate::mi_realloc), but allocates out of
+    /// the specific heap instead of the default.
+    pub fn mi_heap_realloc(heap: *mut mi_heap_t, p: *mut c_void, newsize: usize) -> *mut c_void;
+
+    /// Equivalent to [`mi_reallocn`], but allocates out of the specific heap
+    /// instead of the default.
+    pub fn mi_heap_reallocn(
+        heap: *mut mi_heap_t,
+        p: *mut c_void,
+        count: usize,
+        size: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_reallocf`], but allocates out of the specific heap
+    /// instead of the default.
+    pub fn mi_heap_reallocf(heap: *mut mi_heap_t, p: *mut c_void, newsize: usize) -> *mut c_void;
+
+    /// Equivalent to [`mi_strdup`], but allocates out of the specific heap
+    /// instead of the default.
+    pub fn mi_heap_strdup(heap: *mut mi_heap_t, s: *const c_char) -> *mut c_char;
+
+    /// Equivalent to [`mi_strndup`], but allocates out of the specific heap
+    /// instead of the default.
+    pub fn mi_heap_strndup(heap: *mut mi_heap_t, s: *const c_char, n: usize) -> *mut c_char;
+
+    /// Equivalent to [`mi_realpath`], but allocates out of the specific heap
+    /// instead of the default.
+    pub fn mi_heap_realpath(
+        heap: *mut mi_heap_t,
+        fname: *const c_char,
+        resolved_name: *mut c_char,
+    ) -> *mut c_char;
+
+    /// Equivalent to [`mi_malloc_aligned`](crate::mi_malloc_aligned), but
+    /// allocates out of the specific heap instead of the default.
+    pub fn mi_heap_malloc_aligned(
+        heap: *mut mi_heap_t,
+        size: usize,
+        alignment: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_malloc_aligned_at`], but allocates out of the
+    /// specific heap instead of the default.
+    pub fn mi_heap_malloc_aligned_at(
+        heap: *mut mi_heap_t,
+        size: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_zalloc_aligned`](crate::mi_zalloc_aligned), but
+    /// allocates out of the specific heap instead of the default.
+    pub fn mi_heap_zalloc_aligned(
+        heap: *mut mi_heap_t,
+        size: usize,
+        alignment: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_zalloc_aligned_at`], but allocates out of the
+    /// specific heap instead of the default.
+    pub fn mi_heap_zalloc_aligned_at(
+        heap: *mut mi_heap_t,
+        size: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_calloc_aligned`], but allocates out of the specific
+    /// heap instead of the default.
+    pub fn mi_heap_calloc_aligned(
+        heap: *mut mi_heap_t,
+        count: usize,
+        size: usize,
+        alignment: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_calloc_aligned_at`], but allocates out of the
+    /// specific heap instead of the default.
+    pub fn mi_heap_calloc_aligned_at(
+        heap: *mut mi_heap_t,
+        count: usize,
+        size: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_realloc_aligned`](crate::mi_realloc_aligned), but allocates out of the specific
+    /// heap instead of the default.
+    pub fn mi_heap_realloc_aligned(
+        heap: *mut mi_heap_t,
+        p: *mut c_void,
+        newsize: usize,
+        alignment: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_realloc_aligned_at`], but allocates out of the
+    /// specific heap instead of the default.
+    pub fn mi_heap_realloc_aligned_at(
+        heap: *mut mi_heap_t,
+        p: *mut c_void,
+        newsize: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_rezalloc`], but allocates out of the specific heap
+    /// instead of the default.
+    pub fn mi_heap_rezalloc(heap: *mut mi_heap_t, p: *mut c_void, newsize: usize) -> *mut c_void;
+
+    /// Equivalent to [`mi_recalloc`], but allocates out of the specific heap
+    /// instead of the default.
+    pub fn mi_heap_recalloc(
+        heap: *mut mi_heap_t,
+        p: *mut c_void,
+        newcount: usize,
+        size: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_rezalloc_aligned`], but allocates out of the specific
+    /// heap instead of the default.
+    pub fn mi_heap_rezalloc_aligned(
+        heap: *mut mi_heap_t,
+        p: *mut c_void,
+        newsize: usize,
+        alignment: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_rezalloc_aligned_at`], but allocates out of the
+    /// specific heap instead of the default.
+    pub fn mi_heap_rezalloc_aligned_at(
+        heap: *mut mi_heap_t,
+        p: *mut c_void,
+        newsize: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_recalloc_aligned`], but allocates out of the
+    /// specific heap instead of the default.
+    pub fn mi_heap_recalloc_aligned(
+        heap: *mut mi_heap_t,
+        p: *mut c_void,
+        newcount: usize,
+        size: usize,
+        alignment: usize,
+    ) -> *mut c_void;
+
+    /// Equivalent to [`mi_recalloc_aligned_at`], but allocates out of the
+    /// specific heap instead of the default.
+    pub fn mi_heap_recalloc_aligned_at(
+        heap: *mut mi_heap_t,
+        p: *mut c_void,
+        newcount: usize,
+        size: usize,
+        alignment: usize,
+        offset: usize,
+    ) -> *mut c_void;
+
+    /// Does a heap contain a pointer to a previously allocated block?
+    ///
+    /// `p` must be a pointer to a previously allocated block (in any heap) -- it cannot be some
+    /// random pointer!
+    ///
+    /// Returns `true` if the block pointed to by `p` is in the `heap`.
+    ///
+    /// See [`mi_heap_check_owned`].
+    #[cfg(feature = "v2")]
+    pub fn mi_heap_contains_block(heap: *mut mi_heap_t, p: *const c_void) -> bool;
+
+    /// Check safely if any pointer is part of a heap.
+    ///
+    /// `p` may be any pointer -- not required to be previously allocated by the
+    /// given heap or any other mimalloc heap. Returns `true` if `p` points to a
+    /// block in the given heap, false otherwise.
+    ///
+    /// Note: expensive function, linear in the pages in the heap.
+    ///
+    /// See [`mi_heap_contains_block`], [`mi_heap_get_default`], and
+    /// [`mi_is_in_heap_region`]
+    #[cfg(feature = "v2")]
+    pub fn mi_heap_check_owned(heap: *mut mi_heap_t, p: *const c_void) -> bool;
+
+    /// Check safely if any pointer is part of the default heap of this thread.
+    ///
+    /// `p` may be any pointer -- not required to be previously allocated by the
+    /// default heap for this thread, or any other mimalloc heap. Returns `true`
+    /// if `p` points to a block in the default heap, false otherwise.
+    ///
+    /// Note: expensive function, linear in the pages in the heap.
+    ///
+    /// See [`mi_heap_contains_block`], [`mi_heap_get_default`]
+    pub fn mi_check_owned(p: *const c_void) -> bool;
+
+    /// Visit all areas and blocks in `heap`.
+    ///
+    /// If `visit_all_blocks` is false, the `visitor` is only called once for
+    /// every heap area. If it's true, the `visitor` is also called for every
+    /// allocated block inside every area (with `!block.is_null()`). Return
+    /// `false` from the `visitor` to return early.
+    ///
+    /// `arg` is an extra argument passed into the `visitor`.
+    ///
+    /// Returns `true` if all areas and blocks were visited.
+    ///
+    /// Passing a `None` visitor is allowed, and is a no-op.
+    #[cfg(feature = "v2")]
+    pub fn mi_heap_visit_blocks(
+        heap: *const mi_heap_t,
+        visit_all_blocks: bool,
+        visitor: mi_block_visit_fun,
+        arg: *mut c_void,
+    ) -> bool;
+
+    /// Visit all areas and blocks in `heap`.
+    ///
+    /// If `visit_all_blocks` is false, the `visitor` is only called once for
+    /// every heap area. If it's true, the `visitor` is also called for every
+    /// allocated block inside every area (with `!block.is_null()`). Return
+    /// `false` from the `visitor` to return early.
+    ///
+    /// `arg` is an extra argument passed into the `visitor`.
+    ///
+    /// Returns `true` if all areas and blocks were visited.
+    ///
+    /// Passing a `None` visitor is allowed, and is a no-op.
+    ///
+    /// Note: in v3 the `heap` parameter is non-const compared to v2.
+    #[cfg(not(feature = "v2"))]
+    pub fn mi_heap_visit_blocks(
+        heap: *mut mi_heap_t,
+        visit_all_blocks: bool,
+        visitor: mi_block_visit_fun,
+        arg: *mut c_void,
+    ) -> bool;
+
+    #[cfg(feature = "arena")]
+    /// Create a heap that only allocates in the specified arena
+    pub fn mi_heap_new_in_arena(arena_id: mi_arena_id_t) -> *mut mi_heap_t;
+
+    #[cfg(feature = "arena")]
+    /// Reserve OS memory for use by mimalloc. Reserved areas are used
+    /// before allocating from the OS again. By reserving a large area upfront,
+    /// allocation can be more efficient, and can be better managed on systems
+    /// without `mmap`/`VirtualAlloc` (like WASM for example).
+    ///
+    /// - `size` The size to reserve.
+    /// - `commit` Commit the memory upfront.
+    /// - `allow_large` Allow large OS pages (2MiB) to be used?
+    /// - `exclusive` Only allow allocations if specifically for this arena.
+    /// - `arena_id` Pointer who's value will be set to the new arena_id if successful.
+    ///
+    /// Returns 0 if successful, and an error code otherwise (e.g. `ENOMEM`)
+    pub fn mi_reserve_os_memory_ex(
+        size: usize,
+        commit: bool,
+        allow_large: bool,
+        exclusive: bool,
+        arena_id: *mut mi_arena_id_t,
+    ) -> c_int;
+
+    /// Check if the heap page containing `p` is under-utilized.
+    ///
+    /// # Safety
+    /// Assumes the page belonging to `p` is only accessed by the calling thread.
+    pub fn mi_unsafe_heap_page_is_under_utilized(
+        heap: *mut mi_heap_t,
+        p: *mut c_void,
+        perc_threshold: usize,
+    ) -> bool;
+
+    #[cfg(not(feature = "v2"))]
+    /// Return the minimum size for an arena (v3 only).
+    pub fn mi_arena_min_size() -> usize;
+
+    #[cfg(not(feature = "v2"))]
+    /// Equivalent to [`mi_heap_zalloc_small`], but for a thread-local heap (`theap`) in v3.
+    ///
+    /// `size` must be smaller or equal to [`MI_SMALL_SIZE_MAX`].
+    pub fn mi_theap_zalloc_small(theap: *mut mi_theap_t, size: usize) -> *mut c_void;
+
+    #[cfg(feature = "arena")]
+    /// Manage a particular memory area for use by mimalloc.
+    /// This is just like `mi_reserve_os_memory_ex` except that the area should already be
+    /// allocated in some manner and available for use my mimalloc.
+    ///
+    /// # Safety
+    /// mimalloc will likely segfault when allocating from the arena if the arena `start` & `size`
+    /// aren't aligned with mimalloc's `MI_SEGMENT_ALIGN` (e.g. 32MB on x86_64 machines).
+    ///
+    /// - `start` Start of the memory area
+    /// - `size` The size of the memory area. Must be large than `MI_ARENA_BLOCK_SIZE` (e.g. 64MB
+    ///          on x86_64 machines).
+    /// - `commit` Set true if the memory range is already commited.
+    /// - `is_large` Set true if the memory range consists of large files, or if the memory should
+    ///              not be decommitted or protected (like rdma etc.).
+    /// - `is_zero` Set true if the memory range consists only of zeros.
+    /// - `numa_node` Possible associated numa node or `-1`.
+    /// - `exclusive` Only allow allocations if specifically for this arena.
+    /// - `arena_id` Pointer who's value will be set to the new arena_id if successful.
+    ///
+    /// Returns `true` if arena was successfully allocated
+    pub fn mi_manage_os_memory_ex(
+        start: *const c_void,
+        size: usize,
+        is_committed: bool,
+        is_large: bool,
+        is_zero: bool,
+        numa_node: c_int,
+        exclusive: bool,
+        arena_id: *mut mi_arena_id_t,
+    ) -> bool;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::mi_malloc;
+    use super::*;
+
+    #[test]
+    fn it_calculates_usable_size() {
+        let ptr = unsafe { mi_malloc(32) } as *mut u8;
+        let usable_size = unsafe { mi_usable_size(ptr as *mut c_void) };
+        assert!(
+            usable_size >= 32,
+            "usable_size should at least equal to the allocated size"
+        );
+    }
+
+    #[test]
+    fn runtime_stable_option() {
+        unsafe {
+            assert_eq!(mi_option_get(mi_option_show_errors), 0);
+            mi_option_set(mi_option_show_errors, 1);
+            assert_eq!(mi_option_get(mi_option_show_errors), 1);
+
+            assert_eq!(mi_option_get(mi_option_show_stats), 0);
+            mi_option_set(mi_option_show_stats, 1);
+            assert_eq!(mi_option_get(mi_option_show_stats), 1);
+
+            assert_eq!(mi_option_get(mi_option_verbose), 0);
+            mi_option_set(mi_option_verbose, 1);
+            assert_eq!(mi_option_get(mi_option_verbose), 1);
+        }
+    }
+}
diff --git a/vendor/libmimalloc-sys/src/lib.rs b/vendor/libmimalloc-sys/src/lib.rs
new file mode 100644
index 00000000..ebd069b4
--- /dev/null
+++ b/vendor/libmimalloc-sys/src/lib.rs
@@ -0,0 +1,139 @@
+#![no_std]
+// Copyright 2019 Octavian Oncescu
+
+use core::ffi::c_void;
+
+#[cfg(feature = "extended")]
+mod extended;
+#[cfg(feature = "extended")]
+pub use extended::*;
+
+extern "C" {
+    /// Allocate zero-initialized `size` bytes.
+    ///
+    /// Returns a pointer to newly allocated zero-initialized memory, or null if
+    /// out of memory.
+    pub fn mi_zalloc(size: usize) -> *mut c_void;
+
+    /// Allocate `size` bytes.
+    ///
+    /// Returns pointer to the allocated memory or null if out of memory.
+    /// Returns a unique pointer if called with `size` 0.
+    pub fn mi_malloc(size: usize) -> *mut c_void;
+
+    /// Re-allocate memory to `newsize` bytes.
+    ///
+    /// Return pointer to the allocated memory or null if out of memory. If null
+    /// is returned, the pointer `p` is not freed. Otherwise the original
+    /// pointer is either freed or returned as the reallocated result (in case
+    /// it fits in-place with the new size).
+    ///
+    /// If `p` is null, it behaves as [`mi_malloc`]. If `newsize` is larger than
+    /// the original `size` allocated for `p`, the bytes after `size` are
+    /// uninitialized.
+    pub fn mi_realloc(p: *mut c_void, newsize: usize) -> *mut c_void;
+
+    /// Allocate `size` bytes aligned by `alignment`, initialized to zero.
+    ///
+    /// Return pointer to the allocated memory or null if out of memory.
+    ///
+    /// Returns a unique pointer if called with `size` 0.
+    pub fn mi_zalloc_aligned(size: usize, alignment: usize) -> *mut c_void;
+
+    /// Allocate `size` bytes aligned by `alignment`.
+    ///
+    /// Return pointer to the allocated memory or null if out of memory.
+    ///
+    /// Returns a unique pointer if called with `size` 0.
+    pub fn mi_malloc_aligned(size: usize, alignment: usize) -> *mut c_void;
+
+    /// Re-allocate memory to `newsize` bytes, aligned by `alignment`.
+    ///
+    /// Return pointer to the allocated memory or null if out of memory. If null
+    /// is returned, the pointer `p` is not freed. Otherwise the original
+    /// pointer is either freed or returned as the reallocated result (in case
+    /// it fits in-place with the new size).
+    ///
+    /// If `p` is null, it behaves as [`mi_malloc_aligned`]. If `newsize` is
+    /// larger than the original `size` allocated for `p`, the bytes after
+    /// `size` are uninitialized.
+    pub fn mi_realloc_aligned(p: *mut c_void, newsize: usize, alignment: usize) -> *mut c_void;
+
+    /// Free previously allocated memory.
+    ///
+    /// The pointer `p` must have been allocated before (or be null).
+    pub fn mi_free(p: *mut c_void);
+}
+
+/// When using the `"override"` feature flag, the user wants us to globally
+/// override the system allocator.
+///
+/// However, since we build and link `mimalloc` as a static library/archive,
+/// the linker may decide to not care about our overrides if it can't directly
+/// see references to the symbols, see the following link for details:
+/// <https://maskray.me/blog/2021-06-20-symbol-processing#archive-processing>
+///
+/// This is problematic if `mimalloc` is used from a library that by itself
+/// doesn't allocate, yet invokes other shared libraries that do, since then
+/// the linker wouldn't see any references to `malloc`/`free`, and the symbols
+/// would not be overridden.
+///
+/// To avoid this problem, we make sure that the allocator functions are
+/// visible to the linker.
+///
+/// To avoid this problem, we reference `mi_malloc` in a `#[used]` static.
+/// This makes it known to `rustc`, which will create a reference to it in a
+/// `symbols.o` stub file that is later passed directly to the linker (instead
+/// of being in an archive). See the following link for details on how this
+/// works: <https://github.com/rust-lang/rust/pull/95604>
+///
+/// NOTE: This works because `mimalloc` is compiled into a single object file
+/// in `static.c`. If it was split across multiple files, we'd need to
+/// reference each symbol. See also the comment at the top of `static.c`.
+///
+/// NOTE: On macOS, mimalloc doesn't just override malloc/free, but also
+/// registers itself with the allocator's zone APIs in a ctor
+/// (`_mi_macos_override_malloc`, marked with `__attribute__((constructor))`).
+/// Similarly to above, for the Mach-O linker to actually consider ctors as
+/// "used" when defined in an archive member in a static library, so we need
+/// to explicitly reference something in the object file. The constructor
+/// symbol itself is static, so we can't get a reference to that, so instead
+/// we reference `mi_malloc` here too).
+#[cfg(feature = "override")]
+mod set_up_statics {
+    use super::*;
+    #[used] // Could be `#[used(linker)]` once stable
+    static USED: unsafe extern "C" fn(usize) -> *mut c_void = mi_malloc;
+}
+
+#[cfg(test)]
+mod tests {
+    extern crate libc;
+    use super::*;
+
+    #[test]
+    fn it_frees_memory_malloc() {
+        let ptr = unsafe { mi_malloc_aligned(8, 8) } as *mut u8;
+        unsafe { mi_free(ptr as *mut c_void) };
+    }
+
+    #[test]
+    fn it_frees_memory_zalloc() {
+        let ptr = unsafe { mi_zalloc_aligned(8, 8) } as *mut u8;
+        unsafe { mi_free(ptr as *mut c_void) };
+    }
+
+    #[test]
+    fn it_frees_memory_realloc() {
+        let ptr = unsafe { mi_malloc_aligned(8, 8) } as *mut u8;
+        let ptr = unsafe { mi_realloc_aligned(ptr as *mut c_void, 8, 8) } as *mut u8;
+        unsafe { mi_free(ptr as *mut c_void) };
+    }
+
+    #[cfg(all(feature = "override", target_vendor = "apple"))]
+    #[test]
+    fn mimalloc_and_libc_are_interoperable_when_overridden() {
+        let ptr = unsafe { mi_malloc(42) };
+        unsafe { libc::free(ptr) };
+    }
+}

From a1e8c132830beb2dd35d0a2e96e415fc9e59937a Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Mon, 19 Oct 2026 11:50:30 -0700
Subject: [PATCH 168/173] fix: hold the GIL wherever objects are built or
 released

The cycle collector is process-wide, and its registry spans every
interpreter's objects. Since this branch, a collection reads its
candidates without taking their cells' borrows (the mark walk's
`peek`s), without holding a reference to them (`with_borrowed`), and
the allocation paths append to the collector's deferred and young
lists through guard-free `peek_mut`s. That is sound only if every
thread that touches objects holds the GIL.

Several threads did not. `Interpreter::default` built an interpreter,
tracking thousands of objects, without the GIL, and an interpreter's
fields released its objects without it when it was dropped. The VM
unit tests run in parallel, each building and dropping its own
interpreter, so one thread appended to the deferred list while
another's collection drained it (`index out of bounds: the len is 1700
but the index is 1700` in `defer_container`, a `swap_remove` past the
end, glibc's `malloc(): unaligned tcache chunk detected`), or released
lists that another thread's mark walk was reading (the SIGSEGV in
`traverse_impl` on a garbage child). A panic raised while Cranelift
frames were on the stack aborted with "failed to initiate panic". On
`main` the collector held a reference to each candidate and took cell
borrows, so only the young lists' `peek_mut` raced, which is why the
abort was rarer there.

Take the GIL where objects are touched outside an eval loop:

- `Interpreter::default` holds the GIL while it builds, and the
  interpreter's `Drop` holds it while its fields drop (a trailing
  `GilScope` field gives it back after the others).
- A worker thread takes the GIL before it builds or forks its
  interpreter, not after.
- The tier-2 JIT's thread-local state, whose tables hold functions and
  classes, is released under the GIL when its thread exits. If the GIL
  stays busy for a second, the state is leaked rather than freed, so
  a thread's or the process's exit can't hang on it.
- The test harness's `run` and the unit tests that build objects
  directly hold the GIL (`gil::hold_gil`), as the contract requires.

A new test checks that building and dropping an interpreter waits for
a GIL held by another thread; it fails without the fix.
---
 crates/weavepy-vm/src/gc_trace.rs             |   7 +
 crates/weavepy-vm/src/gil.rs                  | 135 +++++++++++++++++-
 crates/weavepy-vm/src/lib.rs                  |  88 ++++++++++--
 crates/weavepy-vm/src/object.rs               |   6 +
 crates/weavepy-vm/src/specialize.rs           |   3 +
 crates/weavepy-vm/src/stdlib/pickle_accel.rs  |   3 +
 .../src/stdlib/pickle_accel/encode.rs         |   2 +
 crates/weavepy-vm/src/stdlib/thread_real.rs   |  19 +--
 crates/weavepy-vm/src/stdlib/weakref_real.rs  |   2 +
 crates/weavepy-vm/src/sync.rs                 |   5 +
 crates/weavepy-vm/src/tier2.rs                |  15 +-
 11 files changed, 264 insertions(+), 21 deletions(-)

diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs
index e2ee8dff..3497768b 100644
--- a/crates/weavepy-vm/src/gc_trace.rs
+++ b/crates/weavepy-vm/src/gc_trace.rs
@@ -3760,6 +3760,7 @@ mod tests {
 
     #[test]
     fn registry_entries_stay_compact_and_slots_are_reused() {
+        let _gil = crate::gil::hold_gil();
         #[cfg(target_pointer_width = "64")]
         assert_eq!(std::mem::size_of::<Entry>(), 32);
         let state = GcState::new();
@@ -3799,6 +3800,7 @@ mod tests {
 
     #[test]
     fn removals_keep_generation_and_frozen_positions() {
+        let _gil = crate::gil::hold_gil();
         for frozen in [false, true] {
             let state = GcState::new();
             let roots: Vec<_> = (0..4)
@@ -3847,6 +3849,7 @@ mod tests {
 
     #[test]
     fn live_roots_promote_freeze_and_untrack() {
+        let _gil = crate::gil::hold_gil();
         let state = GcState::new();
         let roots: Vec<Object> = (0..4)
             .map(|_| Object::Dict(Rc::new(RefCell::new(DictData::default()))))
@@ -3883,6 +3886,7 @@ mod tests {
 
     #[test]
     fn track_and_untrack() {
+        let _gil = crate::gil::hold_gil();
         let s = GcState::new();
         let d = Object::Dict(Rc::new(RefCell::new(DictData::default())));
         // `track_now`, not `track`: an empty dict holds nothing that could
@@ -3896,6 +3900,7 @@ mod tests {
 
     #[test]
     fn collect_clears_simple_cycle() {
+        let _gil = crate::gil::hold_gil();
         let s = GcState::new();
         let dict = Rc::new(RefCell::new(DictData::default()));
         let outer = Object::Dict(dict.clone());
@@ -3917,6 +3922,7 @@ mod tests {
 
     #[test]
     fn tracked_objects_die_by_refcount() {
+        let _gil = crate::gil::hold_gil();
         let s = GcState::new();
         let dict = Rc::new(RefCell::new(DictData::default()));
         let obj = Object::Dict(dict.clone());
@@ -3933,6 +3939,7 @@ mod tests {
 
     #[test]
     fn freeze_unfreeze_round_trip() {
+        let _gil = crate::gil::hold_gil();
         let s = GcState::new();
         let d = Object::Dict(Rc::new(RefCell::new(DictData::default())));
         // See `track_and_untrack` on `track_now`.
diff --git a/crates/weavepy-vm/src/gil.rs b/crates/weavepy-vm/src/gil.rs
index 41a2749c..6da537fd 100644
--- a/crates/weavepy-vm/src/gil.rs
+++ b/crates/weavepy-vm/src/gil.rs
@@ -298,6 +298,30 @@ impl GilState {
         }
     }
 
+    /// [`Self::acquire`], giving up after `timeout`.
+    pub(crate) fn acquire_within(self: &Arc<Self>, timeout: Duration) -> Option<GilGuard> {
+        if free_threading_enabled() {
+            return Some(self.acquire());
+        }
+        self.breaker.add_waiter();
+        let lock_guard = self.lock.try_lock_for(timeout);
+        self.breaker.remove_waiter();
+        let lock_guard = lock_guard?;
+        self.note_acquired();
+        let me = current_thread_id();
+        crate::sync::note_vm_thread(me);
+        self.holder.store(me, Ordering::Release);
+        self.depth.fetch_add(1, Ordering::AcqRel);
+        // SAFETY: as in `acquire`.
+        let static_guard: parking_lot::ReentrantMutexGuard<'static, ()> =
+            unsafe { std::mem::transmute(lock_guard) };
+        Some(GilGuard {
+            state: Arc::clone(self),
+            _lock_guard: Some(static_guard),
+            lock_free: false,
+        })
+    }
+
     /// Try to acquire without blocking.
     pub fn try_acquire(self: &Arc<Self>) -> Option<GilGuard> {
         if free_threading_enabled() {
@@ -688,6 +712,113 @@ pub fn current_thread_holds_gil() -> bool {
     GIL_GUARD_STACK.with(|cell| !cell.borrow().is_empty())
 }
 
+/// Hold the GIL for a scope: take it (on the guard stack, so the
+/// cooperative hand-off can still pass it on) unless the calling thread
+/// already holds it. The returned [`GilScope`] gives it back on drop.
+///
+/// Every thread that touches VM objects must hold the GIL, not only the
+/// one running bytecode: the cycle collector is process-wide, and a
+/// collection on one thread reads every tracked object, including
+/// another interpreter's, without taking their cells' borrows. Code that
+/// builds, mutates or releases objects outside an eval loop -- an
+/// embedder constructing or dropping an [`crate::Interpreter`], a
+/// thread's teardown, a native unit test -- holds one of these.
+pub fn hold_gil() -> GilScope {
+    let mut scope = GilScope::idle();
+    scope.engage();
+    scope
+}
+
+/// The GIL held for a scope (see [`hold_gil`]).
+#[derive(Debug, Default)]
+#[must_use = "the GIL is released when the scope is dropped"]
+pub struct GilScope {
+    /// Whether this scope pushed a guard it must pop.
+    engaged: bool,
+}
+
+impl GilScope {
+    /// A scope that holds nothing until [`Self::engage`]d.
+    pub const fn idle() -> Self {
+        Self { engaged: false }
+    }
+
+    /// Take the GIL unless this scope or the calling thread already holds
+    /// it.
+    pub fn engage(&mut self) {
+        if self.engaged {
+            return;
+        }
+        // (`try_with`: an interpreter may be dropped by a thread-local
+        // destructor, after this thread's guard stack is gone; there is
+        // nothing to push the guard onto then, and no bytecode can run.)
+        let held = GIL_GUARD_STACK.try_with(|cell| !cell.borrow().is_empty());
+        if held != Ok(false) {
+            return;
+        }
+        push_gil_guard(global_gil_ref().acquire());
+        note_gil_acquired();
+        self.engaged = true;
+    }
+}
+
+impl Drop for GilScope {
+    fn drop(&mut self) {
+        if self.engaged {
+            // The cooperative hand-off may have replaced the stack's
+            // guards meanwhile, but it restores as many as it took.
+            drop(GIL_GUARD_STACK.try_with(|cell| cell.borrow_mut().pop()));
+        }
+    }
+}
+
+/// A thread-local value that holds objects, released under the GIL when
+/// its thread exits.
+///
+/// A thread's destructors run after it has given the GIL back (and after
+/// its guard stack may be gone), so a cache that holds the last reference
+/// to a tracked object would otherwise free it while another thread's
+/// collection reads it. The destructor takes the GIL directly, without
+/// the guard stack, for the length of the release. If the GIL stays busy
+/// for [`EXIT_GIL_WAIT`] (another thread blocked while holding it), the
+/// value is leaked instead: a thread's exit, or the process's, must not
+/// hang on it.
+pub(crate) struct ReleaseUnderGil<T>(std::mem::ManuallyDrop<T>);
+
+/// How long a thread-local destructor waits for the GIL (see
+/// [`ReleaseUnderGil`]).
+const EXIT_GIL_WAIT: Duration = Duration::from_secs(1);
+
+impl<T> ReleaseUnderGil<T> {
+    pub(crate) const fn new(value: T) -> Self {
+        Self(std::mem::ManuallyDrop::new(value))
+    }
+}
+
+impl<T> std::ops::Deref for ReleaseUnderGil<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self.0
+    }
+}
+
+impl<T> Drop for ReleaseUnderGil<T> {
+    fn drop(&mut self) {
+        let gil = global_gil_ref();
+        let _guard = if gil.lock.is_owned_by_current_thread() {
+            None
+        } else {
+            match gil.acquire_within(EXIT_GIL_WAIT) {
+                Some(guard) => Some(guard),
+                None => return,
+            }
+        };
+        // SAFETY: dropped exactly once, here.
+        unsafe { std::mem::ManuallyDrop::drop(&mut self.0) };
+    }
+}
+
 /// RFC 0025: drop the GIL, run `f`, re-acquire the GIL, then
 /// return `f`'s result. Used by every blocking-I/O / lock-acquire
 /// path that would otherwise prevent other threads from running.
@@ -820,8 +951,8 @@ fn no_yield_active() -> bool {
 /// blocking-release re-acquire in [`GilGuard::allow_threads`].
 #[inline]
 pub fn note_gil_acquired() {
-    GIL_HELD_SINCE.with(|c| c.set(Some(std::time::Instant::now())));
-    WAITER_SEEN_SINCE.with(|c| c.set(None));
+    let _ = GIL_HELD_SINCE.try_with(|c| c.set(Some(std::time::Instant::now())));
+    let _ = WAITER_SEEN_SINCE.try_with(|c| c.set(None));
 }
 
 /// Whether a waiter has been waiting on this holder for at least the
diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 5e037829..76650c97 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -1029,20 +1029,33 @@ pub struct Interpreter {
     core_truth: ThreadCell<Option<(u64, NativeTruth)>>,
     /// `WP_DBG_SAMPLE`: periodic frame-entry sampling to stderr.
     dbg_sample: bool,
+    /// The GIL, held while the fields above drop (see the `Drop` impl).
+    /// Last, so that it is released after every other field.
+    drop_gil: crate::gil::GilScope,
+}
+
+impl Drop for Interpreter {
+    fn drop(&mut self) {
+        // The fields release the interpreter's objects: its module cache,
+        // pools and caches hold the last references to most of them, and
+        // tracked objects are visible to every thread's collections. Hold
+        // the GIL while they go, as construction does (`drop_gil` is the
+        // last field, so it gives the GIL back after the others).
+        self.drop_gil.engage();
+    }
 }
 
 impl Default for Interpreter {
     fn default() -> Self {
-        // Register this thread before touching any cell. Construction
-        // populates the *process-global* collector and module cache, and
-        // `GilCell`'s lock-free path is only sound while a single thread
-        // owns every cell — a second thread building its own interpreter
-        // (an embedder, or `cargo test`'s parallel harness) has to flip
-        // cells to shared first, or both take the unlocked path over the
-        // same `GcState` and one loses with `BorrowMutError`. The GIL
-        // acquire path registers for every *later* entry; this covers the
-        // window before a thread has ever held it.
-        crate::sync::note_vm_thread(crate::gil::current_thread_id());
+        // Construction populates the *process-global* collector and module
+        // cache, so it holds the GIL like any other code that touches
+        // objects: a collection running on another thread (another
+        // interpreter's, in an embedder or `cargo test`'s parallel harness)
+        // reads every tracked object and the collector's own lists without
+        // taking their cells' borrows. Taking the GIL also registers this
+        // thread before it touches any cell, which `GilCell`'s lock-free
+        // path requires.
+        let _gil = crate::gil::hold_gil();
         install_parser_unicode_hook();
         // The UCRT's default response to an invalid parameter (e.g. a CRT
         // call on a stale fd) is a *silent* process fast-fail (exit
@@ -1270,6 +1283,7 @@ impl Default for Interpreter {
             core_next: ThreadCell::new(None),
             core_truth: ThreadCell::new(None),
             dbg_sample: crate::hot_gates::env_flags::dbg_sample(),
+            drop_gil: crate::gil::GilScope::idle(),
         };
         // RFC 0025: publish the shared parts of this interpreter
         // (builtins / module cache / stdout / hooks) so workers
@@ -1402,6 +1416,7 @@ impl Interpreter {
             core_next: ThreadCell::new(None),
             core_truth: ThreadCell::new(None),
             dbg_sample: crate::hot_gates::env_flags::dbg_sample(),
+            drop_gil: crate::gil::GilScope::idle(),
         }
     }
 
@@ -80313,6 +80328,7 @@ next(gen)
 
     #[test]
     fn sum_streaming_integer_fastpath_preserves_boundaries_and_fallbacks() {
+        let _gil = crate::gil::hold_gil();
         let zero = Object::Int(0);
         for sequence in [
             Object::new_list(vec![Object::Int(i64::MAX), Object::Int(1), Object::Int(-1)]),
@@ -80340,6 +80356,11 @@ next(gen)
 
     fn run(src: &str) -> String {
         let module = parse_module(src).expect("parse");
+        // Tests run in parallel, each with its own interpreter, and share
+        // the process-wide collector: hold the GIL while this thread's
+        // objects are built, read and released (it still passes the GIL
+        // on at the eval loop's hand-off points).
+        let _gil = crate::gil::hold_gil();
         let code = compile_module(&module).expect("compile");
         let mut interp = Interpreter::new();
         let buf: Rc<RefCell<Vec<u8>>> = Rc::new(RefCell::new(Vec::new()));
@@ -80555,6 +80576,7 @@ assert loop(2000) == 1999000
         std::thread::Builder::new()
             .stack_size(8 * 1024 * 1024)
             .spawn(|| {
+                let _gil = crate::gil::hold_gil();
                 crate::tier2::force_enable_for_test(4);
                 crate::tier2::note_startup_finished();
                 let mut interp = Interpreter::new();
@@ -82257,6 +82279,7 @@ assert loop(2000) == 1999000
     }
 
     fn check_cold_instance_dictionaries() {
+        let _gil = crate::gil::hold_gil();
         let source = r"
 import gc
 class Plain:
@@ -82341,6 +82364,49 @@ assert namespace is exported.__dict__
         .expect("jit worker thread");
     }
 
+    /// Building and dropping an interpreter waits for the GIL. Both
+    /// touch objects of the process-wide collector -- construction tracks
+    /// thousands, and the drop releases them -- and a collection on another
+    /// thread reads every tracked object, and the collector's own lists,
+    /// without taking cell borrows. The parallel unit tests, each with its
+    /// own interpreter, corrupted the collector's deferred list and freed
+    /// lists under a collection's mark walk while they did either outside
+    /// the GIL.
+    #[test]
+    fn interpreters_build_and_drop_under_the_gil() {
+        use std::sync::atomic::{AtomicBool, Ordering};
+        use std::sync::Arc;
+        // Hold the GIL for a while and check that `work`, on another
+        // thread, did not finish meanwhile.
+        fn waits_for_the_gil(work: impl FnOnce() + Send + 'static) {
+            let done = Arc::new(AtomicBool::new(false));
+            let gil = crate::gil::hold_gil();
+            let worker = {
+                let done = done.clone();
+                std::thread::spawn(move || {
+                    work();
+                    done.store(true, Ordering::Release);
+                })
+            };
+            std::thread::sleep(std::time::Duration::from_millis(300));
+            assert!(
+                !done.load(Ordering::Acquire),
+                "the interpreter was built or dropped without the GIL"
+            );
+            drop(gil);
+            worker.join().expect("interpreter thread");
+            assert!(done.load(Ordering::Acquire));
+        }
+        // Warm the process-wide caches, so the timed construction is fast.
+        drop(Interpreter::new());
+        waits_for_the_gil(|| drop(Interpreter::new()));
+        let mut interp = Interpreter::new();
+        let module = parse_module("data = {i: [i] for i in range(100)}\n").expect("parse");
+        let code = compile_module(&module).expect("compile");
+        interp.run_module(&code).expect("run");
+        waits_for_the_gil(move || drop(interp));
+    }
+
     #[test]
     fn runs_print_int() {
         assert_eq!(run("print(42)\n"), "42\n");
@@ -87486,6 +87552,7 @@ print("native pickle coverage: ok")
 
     #[test]
     fn native_leaf_lookup_uses_body_identity() {
+        let _gil = crate::gil::hold_gil();
         let interp = Interpreter::new();
         let receiver = Object::new_list(Vec::new());
         let Object::Builtin(first) = interp.lookup_method(&receiver, "__len__").unwrap() else {
@@ -87507,6 +87574,7 @@ print("native pickle coverage: ok")
 
     #[test]
     fn native_bound_rejection_cache_does_not_own_methods() {
+        let _gil = crate::gil::hold_gil();
         let receiver = Rc::new(RefCell::new(Vec::new()));
         let weak_receiver = Rc::downgrade(&receiver);
         let object = Object::List(receiver);
diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs
index 218ed8ba..65f269a5 100644
--- a/crates/weavepy-vm/src/object.rs
+++ b/crates/weavepy-vm/src/object.rs
@@ -14381,6 +14381,7 @@ mod tests {
 
     #[test]
     fn clone_counts_every_layout() {
+        let _gil = crate::gil::hold_gil();
         use crate::object::Object;
         let tags = [
             (Object::None, 0),
@@ -14428,6 +14429,7 @@ mod tests {
 
     #[test]
     fn small_int_range_collection_preserves_boundary_values() {
+        let _gil = crate::gil::hold_gil();
         for (start, stop, step, expected) in [
             (i64::MAX, i64::MIN, i64::MIN, vec![i64::MAX, -1]),
             (
@@ -14470,6 +14472,7 @@ mod tests {
 
     #[test]
     fn small_int_range_collection_rejects_impossible_capacities() {
+        let _gil = crate::gil::hold_gil();
         // Neither case can attempt a large allocation: the first fails
         // length conversion, and the second exceeds Vec's layout limit.
         for (range, expected) in [
@@ -14537,6 +14540,7 @@ mod tests {
 
     #[test]
     fn truthiness_matches_python_basics() {
+        let _gil = crate::gil::hold_gil();
         assert!(!Object::None.is_truthy());
         assert!(!Object::Bool(false).is_truthy());
         assert!(Object::Bool(true).is_truthy());
@@ -14561,6 +14565,7 @@ mod tests {
 
     #[test]
     fn identity_of_same_list_handle() {
+        let _gil = crate::gil::hold_gil();
         let a = Object::new_list(vec![Object::Int(1)]);
         let b = a.clone();
         let c = Object::new_list(vec![Object::Int(1)]);
@@ -14570,6 +14575,7 @@ mod tests {
 
     #[test]
     fn list_iter_yields_elements_in_order() {
+        let _gil = crate::gil::hold_gil();
         let lst = Object::new_list(vec![Object::Int(1), Object::Int(2), Object::Int(3)]);
         let mut it = lst.make_iter().expect("iter");
         let a = it.next_value().and_then(|x| match x {
diff --git a/crates/weavepy-vm/src/specialize.rs b/crates/weavepy-vm/src/specialize.rs
index 1bf371ee..ee5bafbf 100644
--- a/crates/weavepy-vm/src/specialize.rs
+++ b/crates/weavepy-vm/src/specialize.rs
@@ -1058,6 +1058,7 @@ mod tests {
 
     #[test]
     fn subscr_decisions_cover_ws3_shapes() {
+        let _gil = crate::gil::hold_gil();
         let xs = Object::new_list(vec![Object::Int(1)]);
         let idx = Object::Int(0);
         assert_eq!(
@@ -1094,6 +1095,7 @@ mod tests {
 
     #[test]
     fn store_subscr_decisions_cover_ws3_shapes() {
+        let _gil = crate::gil::hold_gil();
         let xs = Object::new_list(vec![Object::Int(1)]);
         let idx = Object::Int(0);
         assert_eq!(
@@ -1114,6 +1116,7 @@ mod tests {
 
     #[test]
     fn for_iter_str_and_dict_specialize() {
+        let _gil = crate::gil::hold_gil();
         let s_iter = Object::Iter(Rc::new(RefCell::new(PyIterator::Str {
             s: crate::shared_value::SharedStr::from("abc"),
             index: 0,
diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs
index 8ba876f9..9d236854 100644
--- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs
+++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs
@@ -1679,6 +1679,7 @@ mod tests {
 
     #[test]
     fn decoded_strings_keep_utf8_and_memo_aliases() {
+        let _gil = crate::gil::hold_gil();
         let empty = decode(b"\x80\x05\x8c\x00.").unwrap();
         assert!(matches!(empty, Object::Str(value) if value.is_empty()));
         let result = decode(b"\x80\x05]\x94(\x8c\x05\xc3\xa9\xe2\x82\xac\x94h\x01e.").unwrap();
@@ -1693,6 +1694,7 @@ mod tests {
 
     #[test]
     fn decode_scalars_frames_and_memo_aliases() {
+        let _gil = crate::gil::hold_gil();
         assert!(matches!(decode(b"\x80\x04N."), Some(Object::None)));
         let value = decode(b"\x80\x05\x95\x04\0\0\0\0\0\0\0K\x07\x85.").unwrap();
         let Object::Tuple(tuple) = value else {
@@ -1711,6 +1713,7 @@ mod tests {
 
     #[test]
     fn preflight_rejects_cycles_deep_graphs_and_unsupported_keys() {
+        let _gil = crate::gil::hold_gil();
         for data in [
             &b"\x80\x05]\x94h\x00a."[..],
             &b"\x80\x05]\x94(h\x00\x85e."[..],
diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs
index 9159f658..950a7464 100644
--- a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs
+++ b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs
@@ -908,6 +908,7 @@ mod tests {
 
     #[test]
     fn list_batches_preserve_shared_children() {
+        let _gil = crate::gil::hold_gil();
         let child = Object::new_list(vec![Object::Int(7)]);
         let value = Object::new_list(vec![child; 2001]);
         let bytes = encode_with_headroom(&value, 5, 1000).unwrap();
@@ -922,6 +923,7 @@ mod tests {
 
     #[test]
     fn list_encoding_tolerates_concurrent_resizing() {
+        let _gil = crate::gil::hold_gil();
         // Two raw threads share a cell with no GIL between them: the
         // contract `GilCell` documents is that such threads announce
         // themselves before touching objects. (The bias alone is
diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs
index 1aba095a..461145b8 100644
--- a/crates/weavepy-vm/src/stdlib/thread_real.rs
+++ b/crates/weavepy-vm/src/stdlib/thread_real.rs
@@ -1407,6 +1407,16 @@ fn spawn_python_worker(
                 std::thread::yield_now();
             };
             entry.mark_started();
+            // Acquire the process-wide GIL before touching any
+            // Python state -- building the worker's interpreter included
+            // (a fresh one, or a fork of the seed, populates and clones
+            // objects a peer's collection may be reading). Push onto the
+            // shared [`crate::gil::push_gil_guard`] stack so any builtin
+            // we call (e.g. `lock.acquire()`) can drop the GIL via
+            // `allow_threads_then` for the duration of the blocking
+            // section, then re-acquire on return.
+            let gil = crate::gil::global_gil();
+            crate::gil::push_gil_guard(gil.acquire());
             let mut worker_interp = match parent_fork {
                 Some(fork) => {
                     // Forked on the spawning thread; still install the
@@ -1421,14 +1431,6 @@ fn spawn_python_worker(
                     None => crate::Interpreter::new(),
                 },
             };
-            // Acquire the process-wide GIL before touching any
-            // Python state. Push onto the shared
-            // [`crate::gil::push_gil_guard`] stack so any builtin
-            // we call (e.g. `lock.acquire()`) can drop the GIL via
-            // `allow_threads_then` for the duration of the blocking
-            // section, then re-acquire on return.
-            let gil = crate::gil::global_gil();
-            crate::gil::push_gil_guard(gil.acquire());
             let call_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                 worker_interp.call_object(worker_func.clone(), &positional, &kwargs_pairs)
             }));
@@ -1983,6 +1985,7 @@ mod tests {
 
     #[test]
     fn allocate_lock_returns_lock_object() {
+        let _gil = crate::gil::hold_gil();
         let l = allocate_lock(&[]).unwrap();
         match l {
             Object::Instance(inst) => {
diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs
index 4e03fa36..6295abcd 100644
--- a/crates/weavepy-vm/src/stdlib/weakref_real.rs
+++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs
@@ -1426,6 +1426,7 @@ mod tests {
     #[cfg(target_pointer_width = "64")]
     #[test]
     fn native_wrapper_storage_stays_authoritative_after_dict_materialization() {
+        let _gil = crate::gil::hold_gil();
         let target = Object::Type(
             TypeObject::new_user("FixedWeakrefTarget", vec![], DictData::default()).unwrap(),
         );
@@ -1510,6 +1511,7 @@ mod tests {
 
     #[test]
     fn ref_returns_alive_target_then_none_after_clear() {
+        let _gil = crate::gil::hold_gil();
         // Use a supported target: the unchecked string fixture can be
         // cleared by another test's weakref-only collection.
         let target = Object::Type(
diff --git a/crates/weavepy-vm/src/sync.rs b/crates/weavepy-vm/src/sync.rs
index 6fc89b23..6d5b6376 100644
--- a/crates/weavepy-vm/src/sync.rs
+++ b/crates/weavepy-vm/src/sync.rs
@@ -2021,6 +2021,11 @@ impl GilLock {
         self.0.try_lock()
     }
 
+    /// Whether the calling thread holds the lock.
+    pub fn is_owned_by_current_thread(&self) -> bool {
+        self.0.is_owned_by_current_thread()
+    }
+
     pub fn try_lock_for(
         &self,
         timeout: std::time::Duration,
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index eaf41e3b..97a5bb39 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -2517,7 +2517,10 @@ fn native_method_callable(cf: &CompiledFrame, code: &CodeObject) -> bool {
 }
 
 thread_local! {
-    static JIT: RefCell<JitState> = RefCell::new(JitState::new());
+    /// This thread's compiled code and its tables, which hold functions,
+    /// classes and constants: released under the GIL at thread exit.
+    static JIT: crate::gil::ReleaseUnderGil<RefCell<JitState>> =
+        crate::gil::ReleaseUnderGil::new(RefCell::new(JitState::new()));
 
     /// Memoized callee return typing (RFC 0059 WS3 / RFC 0069 WS1):
     /// `(scalar lane, provably-returns-None)`, keyed by code object
@@ -10457,6 +10460,7 @@ mod scalar_update_guard_tests {
 
     #[test]
     fn mutable_descriptor_classes_never_certify_an_update() {
+        let _gil = crate::gil::hold_gil();
         let object = crate::builtin_types::builtin_types().object_.clone();
         let descriptor_type =
             TypeObject::new_user("MutableValue", vec![object.clone()], DictData::default())
@@ -10527,6 +10531,7 @@ mod attr_pin_tests {
 
     #[test]
     fn stale_hints_check_the_current_activation_and_object_identity() {
+        let _gil = crate::gil::hold_gil();
         let class = TypeObject::new_user("PinValue", vec![], DictData::default()).unwrap();
         let a = Object::Instance(Rc::new(crate::types::PyInstance::new(class.clone())));
         let b = Object::Instance(Rc::new(crate::types::PyInstance::new(class)));
@@ -16037,6 +16042,7 @@ mod native_pair_tests {
 
     #[test]
     fn shares_cursors_with_interpreted_steps_and_stays_exhausted() {
+        let _gil = crate::gil::hold_gil();
         let (outer, inner) = enumeration(&[10, 20, 30], -2);
         assert!(matches!(
             inner.borrow_mut().next_value(),
@@ -16065,6 +16071,7 @@ mod native_pair_tests {
 
     #[test]
     fn counter_overflow_leaves_the_value_for_generic_promotion() {
+        let _gil = crate::gil::hold_gil();
         let (outer, inner) = enumeration(&[11, 22, 33], i64::MAX - 1);
         assert_eq!(
             next_enumerated_byte(&outer),
@@ -16094,6 +16101,7 @@ mod native_pair_tests {
 
     #[test]
     fn unsupported_sources_do_not_advance_either_cursor() {
+        let _gil = crate::gil::hold_gil();
         let inner = Rc::new(RefCell::new(PyIterator::Bytes {
             data: crate::shared_value::SharedSlice::from([42_u8].as_slice()),
             index: 0,
@@ -16124,6 +16132,7 @@ mod native_pair_tests {
 
     #[test]
     fn a_live_borrow_does_not_consume_a_byte() {
+        let _gil = crate::gil::hold_gil();
         let (outer, inner) = enumeration(&[42], 0);
         let borrowed = inner.borrow();
         assert_eq!(next_enumerated_byte(&outer), NativeBytePair::Unsupported);
@@ -16146,6 +16155,7 @@ mod native_pair_tests {
 
     #[test]
     fn tuple_pairs_preserve_identity_shared_cursors_and_exhaustion() {
+        let _gil = crate::gil::hold_gil();
         let value = Object::from_str("retained value");
         let (outer, inner) =
             tuple_enumeration(vec![Object::Int(5), value.clone(), Object::None], -2);
@@ -16182,6 +16192,7 @@ mod native_pair_tests {
 
     #[test]
     fn tuple_pin_pressure_preserves_values_and_allows_none() {
+        let _gil = crate::gil::hold_gil();
         let (outer, inner) = tuple_enumeration(vec![Object::Int(42), Object::None], 0);
         assert!(matches!(
             next_enumerated_tuple(&outer, false),
@@ -16207,6 +16218,7 @@ mod native_pair_tests {
 
     #[test]
     fn tuple_mutable_values_and_counter_overflow_keep_generic_continuations() {
+        let _gil = crate::gil::hold_gil();
         let mutable = Object::List(Rc::new(RefCell::new(vec![Object::Int(1)])));
         let (outer, inner) = tuple_enumeration(vec![mutable.clone(), Object::Int(7)], 7);
         assert!(matches!(
@@ -16263,6 +16275,7 @@ mod native_pair_tests {
 
     #[test]
     fn tuple_borrows_and_wrappers_do_not_consume_values() {
+        let _gil = crate::gil::hold_gil();
         let (outer, inner) = tuple_enumeration(vec![Object::Int(42)], 0);
         let borrowed = inner.borrow();
         assert!(matches!(

From dedcad0bf1407452e582d7fee906b940c972130d Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Mon, 19 Oct 2026 11:50:44 -0700
Subject: [PATCH 169/173] fix: isolate the tests that change the recursion
 limit

`sys.setrecursionlimit` sets a process-wide limit (WeavePy runs one
interpreter per process), but the VM unit tests run many interpreters
at once. `jit_native_recursion_limit_matches_interpreter` set the limit
to 100 and left it there, and `recursion::tests` briefly set it to 4.
`jit_direct_method_call_guard_changes_match_interpreter` sets 300 and
prints the depth at which `RecursionError` first fires, once with the
JIT and once without, so a peer test changing the limit midway made
the two runs report different depths (240 vs 297), on `main` too. The
limit-4 window could also make any concurrent test fail at depth 4.

Run the two Python-level tests in a child process of their own, as
other tests that depend on process-wide state already do, and let the
unit test pass its limit to the entry check directly instead of
changing the shared one. The assertions are unchanged.
---
 crates/weavepy-vm/src/lib.rs       | 44 +++++++++++++++++++++++++++---
 crates/weavepy-vm/src/recursion.rs | 19 ++++++++-----
 2 files changed, 52 insertions(+), 11 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index 76650c97..f021e01c 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -82278,6 +82278,29 @@ assert loop(2000) == 1999000
         .expect("jit worker thread")
     }
 
+    /// Run `body` for the test `name` (its path in the test binary) in a
+    /// child process of its own: `sys.setrecursionlimit` sets a
+    /// process-wide limit, which a test must not change under the
+    /// interpreters the other tests run on concurrent threads (or have
+    /// changed under it).
+    #[cfg(feature = "jit")]
+    fn with_own_recursion_limit(name: &str, body: impl FnOnce()) {
+        const CHILD: &str = "WEAVEPY_RECURSION_LIMIT_TEST_CHILD";
+        if std::env::var_os(CHILD).is_some() {
+            body();
+            return;
+        }
+        let status = std::process::Command::new(std::env::current_exe().expect("test executable"))
+            .args(["--exact", name, "--nocapture"])
+            .env(CHILD, "1")
+            .status()
+            .expect("spawn recursion-limit test");
+        assert!(
+            status.success(),
+            "isolated recursion-limit test failed: {status}"
+        );
+    }
+
     fn check_cold_instance_dictionaries() {
         let _gil = crate::gil::hold_gil();
         let source = r"
@@ -85732,6 +85755,14 @@ assert namespace is exported.__dict__
                    sys.setrecursionlimit(limit)\n\
                    print('depth', depth)\n\
                    print(run(c, 3000, rebind, 1000))\n";
+        with_own_recursion_limit(
+            "tests::jit_direct_method_call_guard_changes_match_interpreter",
+            || check_direct_method_call_guard_changes(src),
+        );
+    }
+
+    #[cfg(feature = "jit")]
+    fn check_direct_method_call_guard_changes(src: &str) {
         let (out, compiled, _deopts) = run_jit(src);
         assert!(compiled >= 2, "the method kernels never compiled");
         assert_eq!(out, run(src));
@@ -87093,10 +87124,15 @@ print("native pickle coverage: ok")
                    \x20   return down(n - 1)\n\
                    k = 0\nwhile k < 10:\n    down(5)\n    k = k + 1\n\
                    try:\n    down(100000)\nexcept RecursionError:\n    print('rec ok')\n";
-        let (out, calls, _fallbacks, _deopts) = run_jit_native(src);
-        assert!(calls >= 1, "down never took the native call path");
-        assert_eq!(out, "rec ok\n");
-        assert_eq!(out, run(src));
+        with_own_recursion_limit(
+            "tests::jit_native_recursion_limit_matches_interpreter",
+            || {
+                let (out, calls, _fallbacks, _deopts) = run_jit_native(src);
+                assert!(calls >= 1, "down never took the native call path");
+                assert_eq!(out, "rec ok\n");
+                assert_eq!(out, run(src));
+            },
+        );
     }
 
     #[cfg(feature = "jit")]
diff --git a/crates/weavepy-vm/src/recursion.rs b/crates/weavepy-vm/src/recursion.rs
index 5289976c..80d0b156 100644
--- a/crates/weavepy-vm/src/recursion.rs
+++ b/crates/weavepy-vm/src/recursion.rs
@@ -160,8 +160,14 @@ pub fn depth_cell() -> *const Cell<usize> {
 #[allow(clippy::not_unsafe_ptr_arg_deref)]
 #[inline]
 pub fn enter_with(cell: *const Cell<usize>) -> Enter {
-    let limit = recursion_limit();
-    // SAFETY: this thread's own thread-local, alive for the thread.
+    enter_under(cell, recursion_limit())
+}
+
+/// [`enter_with`] against `limit` rather than the process-wide limit.
+#[inline(always)]
+fn enter_under(cell: *const Cell<usize>, limit: usize) -> Enter {
+    // SAFETY: this thread's own thread-local, alive for the thread (see
+    // `enter_with`).
     let d = unsafe { &*cell };
     let n = d.get() + 1;
     if n > limit {
@@ -231,10 +237,10 @@ mod tests {
 
     #[test]
     fn over_limit_raises_every_time_without_inflating_depth() {
-        // Use a tiny limit on this thread's view via the global atomic.
-        // (Tests run single-threaded per #[test] body.)
-        let saved = recursion_limit();
-        RECURSION_LIMIT.store(4, Ordering::Relaxed);
+        // A tiny limit, for this thread's activations only: the
+        // process-wide limit is shared with the interpreters other tests
+        // run concurrently, which a limit of 4 would break.
+        let enter = || enter_under(depth_cell(), 4);
 
         let mut guards = Vec::new();
         for _ in 0..4 {
@@ -255,6 +261,5 @@ mod tests {
 
         drop(guards);
         assert_eq!(current_depth(), 0);
-        RECURSION_LIMIT.store(saved, Ordering::Relaxed);
     }
 }

From b136b1b3b3c5b3036cc47c875ead172aaef94614 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Mon, 19 Oct 2026 17:02:07 -0700
Subject: [PATCH 170/173] perf: warm the code generator in the background as
 start-up begins

A process's first compile pays the code generator's cold start (its code
paged in, its tables built: about 0.7 ms). Earlier work in this branch
moved the warm-up from start-up to a background thread started when the
first code object was halfway to its compile threshold, so a process
that never compiles wouldn't carry the code generator's pages. But a hot
loop gets from halfway to its compile in far less time than the warm-up
takes, so the first compile still paid most of the cold start on the
thread that needs the code: the bench gate's short JIT fixtures ran
6-11% slower than on main (sumvm, nested_loops, jitloop), and on the
Windows runner sumvm crossed the gate's 15% limit.

The background warm-up now starts as start-up begins (run_site) when the
JIT may run, and the halfway trigger stays as a fallback. It costs the
main thread nothing: `-c pass` takes the same 10 ms. The price is the
code generator's pages, about 2.6 MB resident in a process that never
compiles (12.6 MB against 10.0 MB; main, which warmed synchronously at
start-up, is 11.8 MB).

Against the previous commit (best of 20): sumvm 4.20 -> 3.66 ms, jitloop
8.07 -> 7.44 ms.
---
 crates/weavepy-vm/src/lib.rs   |  2 ++
 crates/weavepy-vm/src/tier2.rs | 24 ++++++++++++++++++------
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs
index f021e01c..6fb1dc46 100644
--- a/crates/weavepy-vm/src/lib.rs
+++ b/crates/weavepy-vm/src/lib.rs
@@ -1909,6 +1909,8 @@ impl Interpreter {
     /// `-S` is active; another explicit call would repeat `.pth` processing.
     /// Startup import errors retain their best-effort handling.
     pub fn run_site(&mut self) -> Result<(), RuntimeError> {
+        #[cfg(feature = "jit")]
+        crate::tier2::prewarm_at_startup();
         #[cfg(feature = "jit")]
         let _compile_guard = crate::tier2::startup_compilation_scope();
         // Startup runs bytecode too (site/.pth imports can allocate enough
diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs
index 97a5bb39..308ce617 100644
--- a/crates/weavepy-vm/src/tier2.rs
+++ b/crates/weavepy-vm/src/tier2.rs
@@ -826,12 +826,9 @@ struct JitState {
 static JIT_PROCESS_GATE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
 
 /// Warm the code generator on a background thread (see
-/// [`prewarm_codegen`]), once per process, when the first code object is
-/// halfway to its compile threshold. Not at start-up: paging the code
-/// generator in costs a program that never compiles anything about 2 MB
-/// of resident memory (a third of an empty script's over CPython's), and
-/// on the development host it slowed start-up more than it saved a short
-/// hot loop's first compile.
+/// [`prewarm_codegen`]), once per process: when start-up begins with the
+/// JIT on (see [`prewarm_at_startup`]), or failing that when the first
+/// code object is halfway to its compile threshold.
 fn spawn_codegen_prewarm() {
     static SPAWNED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
     if SPAWNED.swap(true, std::sync::atomic::Ordering::Relaxed) {
@@ -847,6 +844,21 @@ fn spawn_codegen_prewarm() {
     }
 }
 
+/// Start the code generator's warm-up as start-up begins, when the JIT may
+/// run. A hot loop gets from its first iterations to its compile in far
+/// less time than the warm-up takes (about 0.7 ms), so a warm-up started
+/// on the way there leaves the first compile paying most of the cold start
+/// on the thread that needs the code: short hot programs ran 6-11% slower
+/// for it. On a background thread during start-up it costs the main thread
+/// nothing; the price is the code generator's pages (about 2 MB resident)
+/// in a process that never compiles, which start-up also paid when it
+/// warmed the code generator itself.
+pub(crate) fn prewarm_at_startup() {
+    if !jit_off_for_process() {
+        spawn_codegen_prewarm();
+    }
+}
+
 /// The warm-up thread [`spawn_codegen_prewarm`] started, until joined.
 static PREWARM_THREAD: std::sync::Mutex<Option<std::thread::JoinHandle<()>>> =
     std::sync::Mutex::new(None);

From e8d5065ac91b81392562bdf6f35fb1d5c1fb460f Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Mon, 19 Oct 2026 20:01:19 -0700
Subject: [PATCH 171/173] test: deselect Pillow's AVIF peak-RSS leak test on
 macOS only

Pillow's TestAvifLeaks::test_leak_load bounds the growth of the
process's peak RSS (ru_maxrss) over one AVIF decode at cpu_count * 3 MB,
9 MB on the 3-core macOS runner. It fails whenever the process sits at
its high-water mark as the test starts, and in the ecosystem lane it
does: by then the suite has grown the process to about 3 GB on a 7 GB
runner, and under that memory pressure the peak hugged the working set
(measured margins 0-54 MB), so the decode's own buffers set a new peak
(21.7 MB in a logged failing run).

It isn't a leak. The same binary passed the test in 5 of 5 standalone
runs of the suite on the same runner image, the other four leak tests'
100 iterations grew by 0 KB in the failing run, and it passes on Linux.
main passed for an incidental reason: its collector allocated hundreds
of MB of scratch per full collection (357 MB for 3M containers, against
48 MB on this branch), which set peaks far above the working set.

The harness gains OS-scoped deselect lists (deselect_linux,
deselect_macos, deselect_windows), applied only on that OS, so the test
keeps running on Linux; a test pins the scoping against the checked-in
manifest.
---
 crates/weavepy-conformance/src/ecosystem.rs | 68 ++++++++++++++++++---
 tests/ecosystem/manifest.toml               | 19 ++++++
 2 files changed, 79 insertions(+), 8 deletions(-)

diff --git a/crates/weavepy-conformance/src/ecosystem.rs b/crates/weavepy-conformance/src/ecosystem.rs
index 844ebde5..01e517ec 100644
--- a/crates/weavepy-conformance/src/ecosystem.rs
+++ b/crates/weavepy-conformance/src/ecosystem.rs
@@ -100,7 +100,9 @@ pub struct SelftestSpec {
     pub command: String,
     /// `pytest --deselect` arguments — the measured, enumerated escapes.
     /// Every entry carries an inline manifest comment naming the failure
-    /// class.
+    /// class. Holds `deselect` plus the host OS's `deselect_<os>` list
+    /// (`deselect_linux`, `deselect_macos`, `deselect_windows`), for an
+    /// escape that only one OS's runner needs.
     pub deselect: Vec<String>,
     /// Optional wall budget for the self-test stage alone.
     pub timeout_seconds: Option<u64>,
@@ -311,13 +313,12 @@ impl Manifest {
             .and_then(simple_tables::Value::as_str)
             .map(str::to_owned)
             .ok_or_else(|| anyhow::anyhow!("[{section}] missing command"))?;
-        let deselect = match kv.get("deselect") {
-            Some(simple_tables::Value::List(items)) => items.clone(),
-            Some(simple_tables::Value::Str(_)) => {
-                anyhow::bail!("[{section}] deselect must be an array of strings")
-            }
-            None => Vec::new(),
-        };
+        let mut deselect = parse_deselect(kv, &section, "deselect")?;
+        deselect.extend(parse_deselect(
+            kv,
+            &section,
+            &format!("deselect_{}", std::env::consts::OS),
+        )?);
         let timeout_seconds = parse_timeout(kv, &section)?;
         Ok(SelftestSpec {
             mode,
@@ -331,6 +332,17 @@ impl Manifest {
     }
 }
 
+/// A selftest's `key` deselect list (see [`SelftestSpec::deselect`]).
+fn parse_deselect(kv: &simple_tables::Table, section: &str, key: &str) -> Result<Vec<String>> {
+    match kv.get(key) {
+        Some(simple_tables::Value::List(items)) => Ok(items.clone()),
+        Some(simple_tables::Value::Str(_)) => {
+            anyhow::bail!("[{section}] {key} must be an array of strings")
+        }
+        None => Ok(Vec::new()),
+    }
+}
+
 fn parse_timeout(kv: &simple_tables::Table, section: &str) -> Result<Option<u64>> {
     match kv
         .get("timeout_seconds")
@@ -1949,6 +1961,46 @@ timeout_seconds = 900
         assert_eq!(st.timeout_seconds, Some(900));
     }
 
+    /// `deselect_<os>` adds to `deselect` on that OS only.
+    #[test]
+    fn manifest_parses_os_scoped_deselect() {
+        let host = std::env::consts::OS;
+        let other = if host == "macos" { "linux" } else { "macos" };
+        let body = format!(
+            "[packages.pkg]\nrequirements = \"pkg==1.0\"\nprobe = \"probes/pkg_probe.py\"\n\n\
+             [packages.pkg.selftest]\nsource = \"pkg==1.0\"\ncommand = \"tests\"\n\
+             deselect = [\"tests/test_a.py\"]\n\
+             deselect_{host} = [\"tests/test_host.py::test_x\"]\n\
+             deselect_{other} = [\"tests/test_other.py\"]\n"
+        );
+        let m = Manifest::from_body(&body, Path::new(".")).unwrap();
+        let st = m.rows[0].selftest.as_ref().expect("selftest spec");
+        assert_eq!(
+            st.deselect,
+            vec!["tests/test_a.py", "tests/test_host.py::test_x"]
+        );
+    }
+
+    /// The checked-in manifest parses, and Pillow's macOS-only escape
+    /// applies on macOS alone.
+    #[test]
+    fn checked_in_manifest_scopes_pillow_avif_leak_to_macos() {
+        let path =
+            Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/ecosystem/manifest.toml");
+        let m = Manifest::load(&path).unwrap();
+        let pillow = m
+            .rows
+            .iter()
+            .find(|r| r.name == "pillow")
+            .expect("pillow row");
+        let st = pillow.selftest.as_ref().expect("pillow selftest");
+        let avif = "Tests/test_file_avif.py::TestAvifLeaks::test_leak_load";
+        assert_eq!(
+            st.deselect.iter().any(|d| d == avif),
+            std::env::consts::OS == "macos"
+        );
+    }
+
     /// The RFC 0075 WS9 overlay shape (lxml): installed mode carrying a
     /// pinned `source` whose sdist donates the test subtree.
     #[test]
diff --git a/tests/ecosystem/manifest.toml b/tests/ecosystem/manifest.toml
index e5a526f8..3c9c2d7e 100644
--- a/tests/ecosystem/manifest.toml
+++ b/tests/ecosystem/manifest.toml
@@ -39,6 +39,9 @@
 #   deselect         — array of `pytest --deselect` arguments; every
 #                      entry carries an inline comment naming the
 #                      measured failure class
+#   deselect_linux, deselect_macos, deselect_windows:
+#                      the same, applied only when the harness runs on
+#                      that OS (an escape one OS's runner needs)
 #   timeout_seconds  — optional budget for the selftest stage alone
 #                      (default 600 s)
 #
@@ -349,6 +352,22 @@ requirements = "pytest==8.4.2"
 # FileNotFoundError only when it detects CI (`on_ci()`) — it skips
 # locally, fails on GitHub Actions, and never exercises WeavePy.
 command = "Tests --deselect Tests/test_tiff_crashes.py"
+# TestAvifLeaks::test_leak_load deselected on macOS only. It bounds the
+# growth of the process's *peak* RSS (`ru_maxrss`) over one AVIF decode
+# at cpu_count * 3 MB (9 MB on the 3-core runner), so it fails whenever
+# the process is at its high-water mark when the test starts. By then
+# the suite has grown the process to about 3 GB on a 7 GB runner, and
+# under that memory pressure WeavePy's RSS sat at its peak (the margin
+# measured 0-54 MB across runs), so the decode's own buffers set a new
+# peak (21.7 MB in the failing run). It isn't a leak: the same binary
+# passed it in 5 of 5 standalone runs of the suite on the same runner
+# image, the other leak tests' 100 iterations grew by 0 KB in the
+# failing run, and it passes on Linux. (main passed only because its
+# collector's scratch, hundreds of MB per full collection, set peaks
+# far above the working set.)
+deselect_macos = [
+    "Tests/test_file_avif.py::TestAvifLeaks::test_leak_load",  # macOS runner peak-RSS headroom
+]
 timeout_seconds = 7200
 
 [packages.lxml]

From dc20d4281c9f6a005d97826c85a6fd2b243e2594 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Mon, 19 Oct 2026 21:58:04 -0700
Subject: [PATCH 172/173] Revert "perf: drop purged pages from the resident
 size on macOS"

This reverts commit ba20d325. Vendoring libmimalloc-sys (about 25,000
lines, one patch to mimalloc's macOS purge) is a dependency decision
that deserves its own review rather than riding along with this branch.
It was brought in expecting it to fix Pillow's AVIF leak test on macOS,
which it didn't (that test's failure is the runner's memory pressure; see
the manifest). Its remaining benefit, about 7% lower peak RSS over the
suite at about +0.2% instructions, moves to a separate change.
---
 .gitignore                                    |    4 +-
 Cargo.lock                                    |    2 +
 Cargo.toml                                    |    5 -
 crates/weavepy-cli/src/alloc.rs               |   23 -
 vendor/libmimalloc-sys/Cargo.toml             |   79 -
 vendor/libmimalloc-sys/LICENSE.txt            |   25 -
 vendor/libmimalloc-sys/README.md              |   81 -
 vendor/libmimalloc-sys/build.rs               |  137 -
 .../libmimalloc-sys/c_src/mimalloc/v3/LICENSE |   21 -
 .../mimalloc/v3/include/mimalloc-new-delete.h |   66 -
 .../mimalloc/v3/include/mimalloc-override.h   |   68 -
 .../mimalloc/v3/include/mimalloc-stats.h      |  168 --
 .../c_src/mimalloc/v3/include/mimalloc.h      |  706 -----
 .../mimalloc/v3/include/mimalloc/atomic.h     |  562 ----
 .../c_src/mimalloc/v3/include/mimalloc/bits.h |  340 ---
 .../mimalloc/v3/include/mimalloc/internal.h   | 1282 ---------
 .../c_src/mimalloc/v3/include/mimalloc/prim.h |  542 ----
 .../mimalloc/v3/include/mimalloc/track.h      |  150 -
 .../mimalloc/v3/include/mimalloc/types.h      |  732 -----
 .../c_src/mimalloc/v3/src/alloc-aligned.c     |  441 ---
 .../c_src/mimalloc/v3/src/alloc-override.c    |  396 ---
 .../c_src/mimalloc/v3/src/alloc-posix.c       |  202 --
 .../c_src/mimalloc/v3/src/alloc.c             |  885 ------
 .../c_src/mimalloc/v3/src/arena-meta.c        |  179 --
 .../c_src/mimalloc/v3/src/arena.c             | 2514 -----------------
 .../c_src/mimalloc/v3/src/bitmap.c            | 1949 -------------
 .../c_src/mimalloc/v3/src/bitmap.h            |  343 ---
 .../c_src/mimalloc/v3/src/free.c              |  658 -----
 .../c_src/mimalloc/v3/src/heap.c              |  273 --
 .../c_src/mimalloc/v3/src/init.c              | 1199 --------
 .../c_src/mimalloc/v3/src/libc.c              |  477 ----
 .../c_src/mimalloc/v3/src/options.c           |  704 -----
 .../c_src/mimalloc/v3/src/os.c                |  923 ------
 .../c_src/mimalloc/v3/src/page-map.c          |  447 ---
 .../c_src/mimalloc/v3/src/page-queue.c        |  458 ---
 .../c_src/mimalloc/v3/src/page.c              | 1039 -------
 .../mimalloc/v3/src/prim/emscripten/prim.c    |  261 --
 .../v3/src/prim/osx/alloc-override-zone.c     |  481 ----
 .../c_src/mimalloc/v3/src/prim/osx/prim.c     |    9 -
 .../c_src/mimalloc/v3/src/prim/prim.c         |   76 -
 .../c_src/mimalloc/v3/src/prim/readme.md      |    9 -
 .../c_src/mimalloc/v3/src/prim/unix/prim.c    | 1001 -------
 .../c_src/mimalloc/v3/src/prim/wasi/prim.c    |  291 --
 .../v3/src/prim/windows/etw-mimalloc.wprp     |   61 -
 .../c_src/mimalloc/v3/src/prim/windows/etw.h  |  905 ------
 .../mimalloc/v3/src/prim/windows/etw.man      |  Bin 3926 -> 0 bytes
 .../c_src/mimalloc/v3/src/prim/windows/prim.c | 1124 --------
 .../mimalloc/v3/src/prim/windows/readme.md    |   17 -
 .../c_src/mimalloc/v3/src/random.c            |  258 --
 .../c_src/mimalloc/v3/src/static.c            |   43 -
 .../c_src/mimalloc/v3/src/stats.c             |  813 ------
 .../c_src/mimalloc/v3/src/theap.c             |  715 -----
 .../c_src/mimalloc/v3/src/threadlocal.c       |  236 --
 vendor/libmimalloc-sys/src/extended.rs        | 1126 --------
 vendor/libmimalloc-sys/src/lib.rs             |  139 -
 55 files changed, 3 insertions(+), 25642 deletions(-)
 delete mode 100644 vendor/libmimalloc-sys/Cargo.toml
 delete mode 100644 vendor/libmimalloc-sys/LICENSE.txt
 delete mode 100644 vendor/libmimalloc-sys/README.md
 delete mode 100644 vendor/libmimalloc-sys/build.rs
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/LICENSE
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-new-delete.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-override.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-stats.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/atomic.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/bits.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/internal.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/prim.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/track.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/types.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-aligned.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-override.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-posix.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena-meta.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/free.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/heap.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/init.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/libc.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/options.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/os.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-map.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-queue.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/emscripten/prim.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/alloc-override-zone.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/prim.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/prim.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/readme.md
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/wasi/prim.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw-mimalloc.wprp
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.h
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.man
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/prim.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/readme.md
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/random.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/static.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/stats.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/theap.c
 delete mode 100644 vendor/libmimalloc-sys/c_src/mimalloc/v3/src/threadlocal.c
 delete mode 100644 vendor/libmimalloc-sys/src/extended.rs
 delete mode 100644 vendor/libmimalloc-sys/src/lib.rs

diff --git a/.gitignore b/.gitignore
index 48c1aefb..7735b866 100644
--- a/.gitignore
+++ b/.gitignore
@@ -68,12 +68,10 @@ node_modules/
 # Ignore everything under vendor/ EXCEPT the lightly-patched lzma-sys crate,
 # which is a committed build input wired via Cargo.toml [patch.crates-io],
 # and the expat-sys crate (RFC 0056 WS3: vendored expat 2.6.4 + FFI bindings)
-# that backs the native `pyexpat` module, and the patched libmimalloc-sys
-# crate (see Cargo.toml [patch.crates-io]).
+# that backs the native `pyexpat` module.
 /vendor/*
 !/vendor/lzma-sys/
 !/vendor/expat-sys/
-!/vendor/libmimalloc-sys/
 
 # Instrumented build for scripts/gen-order-file.sh
 target-order/
diff --git a/Cargo.lock b/Cargo.lock
index a12d4cbf..a3fef29c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1265,6 +1265,8 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 [[package]]
 name = "libmimalloc-sys"
 version = "0.1.49"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
 dependencies = [
  "cc",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index c18619c2..b1a7f413 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -209,13 +209,8 @@ proptest    = "1.5"
 # patch only flips on the already-compiled coders; the crate is otherwise byte-
 # for-byte lzma-sys 0.1.20 (heavy non-compiled trees like docs/po/tests/cli are
 # trimmed). The override applies transitively, so `xz2` picks it up too.
-#
-# It also vendors `libmimalloc-sys` 0.1.49 with a one-function macOS patch:
-# a purge maps fresh pages over the freed range, so freed memory leaves the
-# resident size (see `vendor/libmimalloc-sys/README.md` for why and the diff).
 [patch.crates-io]
 lzma-sys = { path = "vendor/lzma-sys" }
-libmimalloc-sys = { path = "vendor/libmimalloc-sys" }
 
 [workspace.lints.rust]
 unsafe_op_in_unsafe_fn = "warn"
diff --git a/crates/weavepy-cli/src/alloc.rs b/crates/weavepy-cli/src/alloc.rs
index 1fafd60f..a3a9dc22 100644
--- a/crates/weavepy-cli/src/alloc.rs
+++ b/crates/weavepy-cli/src/alloc.rs
@@ -84,28 +84,9 @@ extern "C" {
     fn mi_option_set(option: c_int, value: c_long);
 }
 
-/// mimalloc 3's `mi_option_purge_delay` (its position in `mi_option_t`).
-#[cfg(target_os = "macos")]
-const MI_OPTION_PURGE_DELAY: c_int = 15;
-
 /// mimalloc 3's `mi_option_arena_reserve` (its position in `mi_option_t`).
 const MI_OPTION_ARENA_RESERVE: c_int = 23;
 
-/// How long freed pages wait before mimalloc purges them on macOS, in
-/// milliseconds, unless the environment sets `MIMALLOC_PURGE_DELAY`.
-///
-/// A purge there maps fresh pages over the range (the vendored crate's
-/// patch, see `vendor/libmimalloc-sys/README.md`), so it is what takes
-/// freed memory out of the resident size. (Elsewhere mimalloc's own
-/// default stands.)
-/// mimalloc's default waits 1000 ms, longer than a short program's whole
-/// warm-up: the transient memory of each JIT compile, and of the program's
-/// own bursts, stayed resident under every later peak (chaos peaked at
-/// 29 MB against 25 MB). The short wait still spares the steady churn of a
-/// loop that frees and reallocates the same pages from repeated faults.
-#[cfg(target_os = "macos")]
-const PURGE_DELAY_MS: c_long = 10;
-
 /// The size of each arena mimalloc reserves, in KiB, unless the
 /// environment sets `MIMALLOC_ARENA_RESERVE`.
 ///
@@ -129,9 +110,5 @@ pub extern "C" fn configure() {
         if libc::getenv(c"MIMALLOC_ARENA_RESERVE".as_ptr()).is_null() {
             mi_option_set(MI_OPTION_ARENA_RESERVE, ARENA_RESERVE_KIB);
         }
-        #[cfg(target_os = "macos")]
-        if libc::getenv(c"MIMALLOC_PURGE_DELAY".as_ptr()).is_null() {
-            mi_option_set(MI_OPTION_PURGE_DELAY, PURGE_DELAY_MS);
-        }
     }
 }
diff --git a/vendor/libmimalloc-sys/Cargo.toml b/vendor/libmimalloc-sys/Cargo.toml
deleted file mode 100644
index db1d6f65..00000000
--- a/vendor/libmimalloc-sys/Cargo.toml
+++ /dev/null
@@ -1,79 +0,0 @@
-# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
-#
-# When uploading crates to the registry Cargo will automatically
-# "normalize" Cargo.toml files for maximal compatibility
-# with all versions of Cargo and also rewrite `path` dependencies
-# to registry (e.g., crates.io) dependencies.
-#
-# If you are reading this file be aware that the original Cargo.toml
-# will likely look very different (and much more reasonable).
-# See Cargo.toml.orig for the original contents.
-
-[package]
-edition = "2018"
-name = "libmimalloc-sys"
-version = "0.1.49"
-authors = ["Octavian Oncescu <octavonce@gmail.com>"]
-build = "build.rs"
-links = "mimalloc"
-exclude = [
-    "/c_src/mimalloc/v2/bin",
-    "/c_src/mimalloc/v2/cmake",
-    "/c_src/mimalloc/v2/doc",
-    "/c_src/mimalloc/v2/docs",
-    "/c_src/mimalloc/v2/ide",
-    "/c_src/mimalloc/v2/test",
-    "/c_src/mimalloc/v3/bin",
-    "/c_src/mimalloc/v3/cmake",
-    "/c_src/mimalloc/v3/doc",
-    "/c_src/mimalloc/v3/docs",
-    "/c_src/mimalloc/v3/ide",
-    "/c_src/mimalloc/v3/test",
-]
-autolib = false
-autobins = false
-autoexamples = false
-autotests = false
-autobenches = false
-description = "Sys crate wrapping the mimalloc allocator"
-readme = false
-keywords = [
-    "allocator",
-    "encrypted-heap",
-    "performance",
-]
-categories = [
-    "memory-management",
-    "api-bindings",
-]
-license = "MIT"
-repository = "https://github.com/purpleprotocol/mimalloc_rust/tree/master/libmimalloc-sys"
-
-[package.metadata.docs.rs]
-features = ["extended"]
-
-[features]
-arena = []
-debug = []
-debug_in_debug = []
-extended = ["cty"]
-local_dynamic_tls = []
-no_thp = []
-override = []
-secure = []
-v2 = []
-win_direct_tls = []
-
-[lib]
-name = "libmimalloc_sys"
-path = "src/lib.rs"
-
-[dependencies.cty]
-version = "0.2"
-optional = true
-
-[dev-dependencies.libc]
-version = "0.2"
-
-[build-dependencies.cc]
-version = "1.2"
diff --git a/vendor/libmimalloc-sys/LICENSE.txt b/vendor/libmimalloc-sys/LICENSE.txt
deleted file mode 100644
index 9a3377d9..00000000
--- a/vendor/libmimalloc-sys/LICENSE.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-Copyright 2019 Octavian Oncescu
-
-Permission is hereby granted, free of charge, to any
-person obtaining a copy of this software and associated
-documentation files (the "Software"), to deal in the
-Software without restriction, including without
-limitation the rights to use, copy, modify, merge,
-publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software
-is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice
-shall be included in all copies or substantial portions
-of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
-ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
-TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
-PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
-IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/vendor/libmimalloc-sys/README.md b/vendor/libmimalloc-sys/README.md
deleted file mode 100644
index 6483af31..00000000
--- a/vendor/libmimalloc-sys/README.md
+++ /dev/null
@@ -1,81 +0,0 @@
-# Vendored `libmimalloc-sys`
-
-This is `libmimalloc-sys` 0.1.49 from crates.io (bundling mimalloc 3.3.2,
-`MI_MALLOC_VERSION 30302`) with one patch. The workspace's
-`[patch.crates-io]` table points at it, so the `weavepy` binary's allocator
-(`crates/weavepy-cli/src/alloc.rs`) builds from here on every platform.
-
-## Why it's vendored
-
-mimalloc returns freed memory to the system by purging it. On macOS,
-upstream purges with `madvise(MADV_FREE_REUSABLE)`, which lowers the
-process's footprint, but the pages stay in its resident size (and so in the
-peak `ru_maxrss`) until the system reclaims them under memory pressure.
-`MADV_DONTNEED` and `MADV_FREE` behave the same way. Only unmapping the
-range, or mapping fresh pages over it, takes them out. With 64 MB of touched
-pages, the resident size stayed at 66 MB after each of those advices and
-fell to 0.6 MB once fresh pages were mapped over the range.
-
-With the upstream purge, nothing freed ever left a WeavePy process's RSS,
-and no runtime option helps. `purge_decommits`, `purge_delay`,
-`page_full_retain`, `page_commit_on_demand` and `arena_reserve` made no
-difference. `disallow_arena_alloc` gets the RSS back, but it maps and
-unmaps every page: 0.55% more instructions over the benchmark suite. The
-patch, with a 10 ms purge delay set in `alloc.rs` (macOS only), brought the
-suite's peak RSS down about 7% for 0.22% more instructions.
-
-On Linux, mimalloc already purges with `MADV_DONTNEED`, which drops resident
-pages there, and Windows decommits. Neither path changes.
-
-## What differs from upstream
-
-- `c_src/mimalloc/v3/src/prim/unix/prim.c`: the patch below.
-- Removed, as nothing here compiles them: mimalloc 2's sources
-  (`c_src/mimalloc/v2`, so the crate's `v2` feature no longer builds) and,
-  under `c_src/mimalloc/v3`, everything but `include/`, `src/` and
-  `LICENSE`. Also removed: the published crate's `Cargo.toml.orig`,
-  `Cargo.lock`, `.cargo_vcs_info.json` and `.gitignore`.
-
-Everything else (`Cargo.toml`, `build.rs`, `src/`, `LICENSE.txt` and the
-compiled C sources) is byte-for-byte upstream.
-
-```diff
---- libmimalloc-sys-0.1.49/c_src/mimalloc/v3/src/prim/unix/prim.c
-+++ vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c
-@@ -492,10 +492,16 @@
- 
- int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
-   int err = 0;
-   #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
--    // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)
--    err = unix_madvise(start, size, MADV_FREE_REUSABLE);
--    if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
-+    // decommit on macOS: map fresh zero pages over the range (WeavePy patch).
-+    // MADV_FREE_REUSABLE only lowers the footprint: the pages stay in the
-+    // resident size (and `ru_maxrss`) until the system reclaims them. If the
-+    // mapping fails, the range is left as it was and marked reusable.
-+    void* p = mmap(start, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, unix_mmap_fd(), 0);
-+    if (p != start) {
-+      err = unix_madvise(start, size, MADV_FREE_REUSABLE);
-+      if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
-+    }
-   #else
-     // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
-     err = unix_madvise(start, size, MADV_DONTNEED);
-```
-
-The mapping keeps the range readable and writable, as upstream's purge
-does, so `needs_recommit` stays false in release builds. A purged range's
-contents are undefined to mimalloc either way, so zero pages are a valid
-result. The mapping uses mimalloc's own VM tag (`unix_mmap_fd()`).
-
-## Updating
-
-To move to a newer `libmimalloc-sys`:
-
-1. Copy the published crate over this directory.
-2. Remove the files listed above.
-3. Reapply the patch to `_mi_prim_decommit`, and update this file.
-
-Cargo doesn't rebuild the C sources when only they change, so touch
-`build.rs` after editing them.
diff --git a/vendor/libmimalloc-sys/build.rs b/vendor/libmimalloc-sys/build.rs
deleted file mode 100644
index 0f3fcda9..00000000
--- a/vendor/libmimalloc-sys/build.rs
+++ /dev/null
@@ -1,137 +0,0 @@
-use std::env;
-use std::fs;
-use std::path::{Path, PathBuf};
-
-fn main() {
-    let mut build = cc::Build::new();
-
-    let version = if env::var("CARGO_FEATURE_V2").is_ok() {
-        "v2"
-    } else {
-        "v3"
-    };
-
-    let cargo_manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set");
-    let include_root = Path::new(&cargo_manifest_dir)
-        .join("c_src")
-        .join("mimalloc")
-        .join(version);
-    let include_dir = include_root
-        .join("include")
-        .to_str()
-        .expect("include path is not valid UTF-8")
-        .to_string();
-    let static_source = include_root.join("src").join("static.c");
-
-    // Make the include directory available to consumers via the `DEP_MIMALLOC_INCLUDE_DIR`
-    // environment variable.
-    println!("cargo:INCLUDE_DIR={include_dir}");
-
-    build.include(format!("c_src/mimalloc/{version}/include"));
-    build.include(format!("c_src/mimalloc/{version}/src"));
-
-    let target_os = env::var("CARGO_CFG_TARGET_OS").expect("target_os not defined!");
-    let target_family = env::var("CARGO_CFG_TARGET_FAMILY").expect("target_family not defined!");
-    let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default();
-    let target_vendor = env::var("CARGO_CFG_TARGET_VENDOR").expect("target_vendor not defined!");
-    let target_arch = env::var("CARGO_CFG_TARGET_ARCH").expect("target_arch not defined!");
-    let cargo_debug = env::var("DEBUG")
-        .map(|value| value == "true" || value == "1")
-        .unwrap_or(false);
-    let debug_enabled = env::var_os("CARGO_FEATURE_DEBUG").is_some()
-        || (env::var_os("CARGO_FEATURE_DEBUG_IN_DEBUG").is_some() && cargo_debug);
-
-    if target_family != "windows" {
-        build.flag("-Wno-error=date-time");
-    }
-
-    if target_env == "msvc" {
-        // Mimalloc expects the MSVC/clang-cl build to use the C++ atomics path.
-        build.cpp(true);
-        build.std("c++17");
-        build.flag_if_supported("/Zc:__cplusplus");
-
-        let wrapper =
-            PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")).join("mimalloc-static.cc");
-        let include = static_source.to_string_lossy().replace('\\', "/");
-        fs::write(&wrapper, format!("#include \"{include}\"\n"))
-            .expect("failed to write mimalloc C++ wrapper");
-        build.file(wrapper);
-    } else {
-        build.file(&static_source);
-    }
-
-    let compiler = build.get_compiler();
-
-    if env::var_os("CARGO_FEATURE_OVERRIDE").is_some() {
-        // Overriding malloc is only available on windows in shared mode, but we
-        // only ever build a static lib.
-        if target_family != "windows" {
-            build.define("MI_MALLOC_OVERRIDE", None);
-        }
-        if target_vendor == "apple" {
-            build.define("MI_OSX_ZONE", Some("1"));
-            build.define("MI_OSX_INTERPOSE", Some("1"));
-        }
-        if !compiler.is_like_msvc() {
-            build.flag_if_supported("-fno-builtin-malloc");
-        }
-    }
-
-    if env::var_os("CARGO_FEATURE_SECURE").is_some() {
-        build.define("MI_SECURE", "4");
-    }
-
-    if target_os == "windows" && env::var_os("CARGO_FEATURE_WIN_DIRECT_TLS").is_some() {
-        build.define("MI_WIN_DIRECT_TLS", "1");
-    }
-
-    let dynamic_tls = env::var("CARGO_FEATURE_LOCAL_DYNAMIC_TLS").is_ok();
-
-    if target_family == "unix" && target_os != "haiku" {
-        if dynamic_tls {
-            build.flag_if_supported("-ftls-model=local-dynamic");
-        } else {
-            build.flag_if_supported("-ftls-model=initial-exec");
-        }
-    }
-
-    if (target_os == "linux" || target_os == "android")
-        && env::var_os("CARGO_FEATURE_NO_THP").is_some()
-    {
-        build.define("MI_NO_THP", "1");
-    }
-
-    if debug_enabled {
-        build.define("MI_DEBUG", "3");
-        build.define("MI_SHOW_ERRORS", "1");
-    } else {
-        // Remove heavy debug assertions etc.
-        build.define("MI_DEBUG", "0");
-        if !cargo_debug {
-            build.define("MI_BUILD_RELEASE", None);
-            build.define("NDEBUG", None);
-        }
-    }
-
-    build.compile("mimalloc");
-
-    // on armv6 we need to link with libatomic
-    if target_os == "linux" && target_arch == "arm" {
-        // Embrace the atomic capability library across various platforms.
-        // For instance, on certain platforms, llvm has relocated the atomic of the arm32 architecture to libclang_rt.builtins.a
-        // while some use libatomic.a, and others use libatomic_ops.a.
-        let atomic_name = env::var("DEP_ATOMIC").unwrap_or("atomic".to_owned());
-        println!("cargo:rustc-link-lib={}", atomic_name);
-    }
-
-    // Link with libs needed on Windows
-    if target_os == "windows" {
-        // https://github.com/microsoft/mimalloc/blob/af21001f7a65eafb8fb16460b018ebf9d75e2ad8/CMakeLists.txt#L487
-        let libs = ["psapi", "shell32", "user32", "advapi32", "bcrypt"];
-
-        for lib in libs {
-            println!("cargo:rustc-link-lib={}", lib);
-        }
-    }
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/LICENSE b/vendor/libmimalloc-sys/c_src/mimalloc/v3/LICENSE
deleted file mode 100644
index 53315ebe..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/LICENSE
+++ /dev/null
@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-new-delete.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-new-delete.h
deleted file mode 100644
index c16f4a66..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-new-delete.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MIMALLOC_NEW_DELETE_H
-#define MIMALLOC_NEW_DELETE_H
-
-// ----------------------------------------------------------------------------
-// This header provides convenient overrides for the new and
-// delete operations in C++.
-//
-// This header should be included in only one source file!
-//
-// On Windows, or when linking dynamically with mimalloc, these
-// can be more performant than the standard new-delete operations.
-// See <https://en.cppreference.com/w/cpp/memory/new/operator_new>
-// ---------------------------------------------------------------------------
-#if defined(__cplusplus)
-  #include <new>
-  #include <mimalloc.h>
-
-  #if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_)
-  // stay consistent with VCRT definitions
-  #define mi_decl_new(n)          mi_decl_nodiscard mi_decl_restrict _Ret_notnull_ _Post_writable_byte_size_(n)
-  #define mi_decl_new_nothrow(n)  mi_decl_nodiscard mi_decl_restrict _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n)
-  #else
-  #define mi_decl_new(n)          mi_decl_nodiscard mi_decl_restrict
-  #define mi_decl_new_nothrow(n)  mi_decl_nodiscard mi_decl_restrict
-  #endif
-
-  void operator delete(void* p) noexcept              { mi_free(p); };
-  void operator delete[](void* p) noexcept            { mi_free(p); };
-
-  void operator delete  (void* p, const std::nothrow_t&) noexcept { mi_free(p); }
-  void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }
-
-  mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }
-  mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }
-
-  mi_decl_new_nothrow(n) void* operator new  (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }
-  mi_decl_new_nothrow(n) void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }
-
-  #if (__cplusplus >= 201402L || _MSC_VER >= 1916)
-  void operator delete  (void* p, std::size_t n) noexcept { mi_free_size(p,n); };
-  void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); };
-  #endif
-
-  #if (__cplusplus > 201402L || defined(__cpp_aligned_new))
-  void operator delete  (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
-  void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
-  void operator delete  (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
-  void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
-  void operator delete  (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
-  void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
-
-  void* operator new  (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
-  void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
-  void* operator new  (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
-  void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
-  #endif
-#endif
-
-#endif // MIMALLOC_NEW_DELETE_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-override.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-override.h
deleted file mode 100644
index 46e05452..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-override.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MIMALLOC_OVERRIDE_H
-#define MIMALLOC_OVERRIDE_H
-
-/* ----------------------------------------------------------------------------
-This header can be used to statically redirect malloc/free and new/delete
-to the mimalloc variants. This can be useful if one can include this file on
-each source file in a project (but be careful when using external code to
-not accidentally mix pointers from different allocators).
------------------------------------------------------------------------------*/
-
-#include <mimalloc.h>
-
-// Standard C allocation
-#define malloc(n)               mi_malloc(n)
-#define calloc(c,n)             mi_calloc(c,n)
-#define realloc(p,n)            mi_realloc(p,n)
-#define free(p)                 mi_free(p)
-
-#define strdup(s)               mi_strdup(s)
-#define strndup(s,n)            mi_strndup(s,n)
-#define realpath(f,n)           mi_realpath(f,n)
-
-// Microsoft extensions
-#define _expand(p,n)            mi_expand(p,n)
-#define _msize(p)               mi_usable_size(p)
-#define _recalloc(p,c,n)        mi_recalloc(p,c,n)
-
-#define _strdup(s)              mi_strdup(s)
-#define _strndup(s,n)           mi_strndup(s,n)
-#define _wcsdup(s)              mi_wcsdup(s)
-#define _mbsdup(s)              mi_mbsdup(s)
-#define _dupenv_s(buf,n,nm)     mi_dupenv_s(buf,n,nm)
-#define _wdupenv_s(buf,n,nm)    mi_wdupenv_s(buf,n,nm)
-
-// Various Posix and Unix variants
-#define reallocf(p,n)           mi_reallocf(p,n)
-#define malloc_size(p)          mi_usable_size(p)
-#define malloc_usable_size(p)   mi_usable_size(p)
-#define malloc_good_size(n)     mi_malloc_good_size(n)
-#define cfree(p)                mi_free(p)
-
-#define valloc(n)               mi_valloc(n)
-#define pvalloc(n)              mi_pvalloc(n)
-#define reallocarray(p,c,n)     mi_reallocarray(p,c,n)
-#define reallocarr(ptrp,c,n)    mi_reallocarr(ptrp,c,n)
-#define memalign(a,n)           mi_memalign(a,n)
-#define aligned_alloc(a,n)      mi_aligned_alloc(a,n)
-#define posix_memalign(p,a,n)   mi_posix_memalign(p,a,n)
-#define _posix_memalign(p,a,n)  mi_posix_memalign(p,a,n)
-
-// Microsoft aligned variants
-#define _aligned_malloc(n,a)                  mi_malloc_aligned(n,a)
-#define _aligned_realloc(p,n,a)               mi_realloc_aligned(p,n,a)
-#define _aligned_recalloc(p,c,n,a)            mi_aligned_recalloc(p,c,n,a)
-#define _aligned_msize(p,a,o)                 mi_usable_size(p)
-#define _aligned_free(p)                      mi_free(p)
-#define _aligned_offset_malloc(n,a,o)         mi_malloc_aligned_at(n,a,o)
-#define _aligned_offset_realloc(p,n,a,o)      mi_realloc_aligned_at(p,n,a,o)
-#define _aligned_offset_recalloc(p,c,n,a,o)   mi_recalloc_aligned_at(p,c,n,a,o)
-
-#endif // MIMALLOC_OVERRIDE_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-stats.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-stats.h
deleted file mode 100644
index 1c620fd1..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc-stats.h
+++ /dev/null
@@ -1,168 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2024-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MIMALLOC_STATS_H
-#define MIMALLOC_STATS_H
-
-#include <mimalloc.h>
-#include <string.h>   // memset
-#include <stdint.h>   // int64_t
-
-#define MI_STAT_VERSION   5  // increased on every backward incompatible change
-
-// alignment for atomic fields
-#if defined(_MSC_VER)
-#define mi_decl_align(a)        __declspec(align(a))
-#elif defined(__GNUC__)
-#define mi_decl_align(a)        __attribute__((aligned(a)))
-#elif __cplusplus >= 201103L
-#define mi_decl_align(a)        alignas(a)
-#else
-#define mi_decl_align(a)
-#endif
-
-
-// count allocation over time
-typedef struct mi_stat_count_s {
-  int64_t total;                              // total allocated
-  int64_t peak;                               // peak allocation
-  int64_t current;                            // current allocation
-} mi_stat_count_t;
-
-// counters only increase
-typedef struct mi_stat_counter_s {
-  int64_t total;                              // total count
-} mi_stat_counter_t;
-
-#define MI_STAT_FIELDS() \
-  MI_STAT_COUNT(pages)                      /* count of mimalloc pages */ \
-  MI_STAT_COUNT(reserved)                   /* reserved memory bytes */ \
-  MI_STAT_COUNT(committed)                  /* committed bytes */ \
-  MI_STAT_COUNTER(reset)                    /* reset bytes */ \
-  MI_STAT_COUNTER(purged)                   /* purged bytes */ \
-  MI_STAT_COUNT(page_committed)             /* committed memory inside pages */ \
-  MI_STAT_COUNT(pages_abandoned)            /* abandoned pages count */ \
-  MI_STAT_COUNT(threads)                    /* number of threads */ \
-  MI_STAT_COUNT(malloc_normal)              /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \
-  MI_STAT_COUNT(malloc_huge)                /* allocated bytes in huge pages */ \
-  MI_STAT_COUNT(malloc_requested)           /* malloc requested bytes */ \
-  \
-  MI_STAT_COUNTER(mmap_calls) \
-  MI_STAT_COUNTER(commit_calls) \
-  MI_STAT_COUNTER(reset_calls) \
-  MI_STAT_COUNTER(purge_calls) \
-  MI_STAT_COUNTER(arena_count)              /* number of memory arena's */ \
-  MI_STAT_COUNTER(malloc_normal_count)      /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \
-  MI_STAT_COUNTER(malloc_huge_count)        /* number of huge bloks */ \
-  MI_STAT_COUNTER(malloc_guarded_count)     /* number of allocations with guard pages */ \
-  \
-  /* internal statistics */ \
-  MI_STAT_COUNTER(arena_rollback_count) \
-  MI_STAT_COUNTER(arena_purges) \
-  MI_STAT_COUNTER(pages_extended)           /* number of page extensions */ \
-  MI_STAT_COUNTER(pages_retire)             /* number of pages that are retired */ \
-  MI_STAT_COUNTER(page_searches)            /* total pages searched for a fresh page */ \
-  MI_STAT_COUNTER(page_searches_count)      /* searched count for a fresh page */ \
-  /* only on v1 and v2 */ \
-  MI_STAT_COUNT(segments) \
-  MI_STAT_COUNT(segments_abandoned) \
-  MI_STAT_COUNT(segments_cache) \
-  MI_STAT_COUNT(_segments_reserved) \
-  /* only on v3 */ \
-  MI_STAT_COUNT(heaps) \
-  MI_STAT_COUNT(theaps) \
-  MI_STAT_COUNTER(pages_reclaim_on_alloc) \
-  MI_STAT_COUNTER(pages_reclaim_on_free) \
-  MI_STAT_COUNTER(pages_reabandon_full) \
-  MI_STAT_COUNTER(pages_unabandon_busy_wait) \
-  MI_STAT_COUNTER(heaps_delete_wait)
-
-// Size bins for chunks
-typedef enum mi_chunkbin_e {
-  MI_CBIN_SMALL,    // slice_count == 1
-  MI_CBIN_OTHER,    // slice_count: any other from the other bins, and 1 <= slice_count <= MI_BCHUNK_BITS
-  MI_CBIN_MEDIUM,   // slice_count == 8
-  MI_CBIN_LARGE,    // slice_count == MI_SIZE_BITS  (only used if MI_ENABLE_LARGE_PAGES is 1)
-  MI_CBIN_HUGE,     // slice_count > MI_BCHUNK_BITS
-  MI_CBIN_NONE,     // no bin assigned yet (the chunk is completely free)
-  MI_CBIN_COUNT
-} mi_chunkbin_t;
-
-
-// Define the statistics structure
-#define MI_BIN_HUGE             (73U)   // see types.h
-#define MI_STAT_COUNT(stat)     mi_stat_count_t stat;
-#define MI_STAT_COUNTER(stat)   mi_stat_counter_t stat;
-
-typedef struct mi_stats_s
-{
-  size_t size;          // size of the mi_stats_t structure 
-  size_t version;       
-
-  mi_decl_align(8)  MI_STAT_FIELDS()
-
-  // future extension
-  mi_stat_count_t   _stat_reserved[4];
-  mi_stat_counter_t _stat_counter_reserved[4];
-
-  // size segregated statistics
-  mi_stat_count_t   malloc_bins[MI_BIN_HUGE+1];   // allocation per size bin
-  mi_stat_count_t   page_bins[MI_BIN_HUGE+1];     // pages allocated per size bin
-  mi_stat_count_t   chunk_bins[MI_CBIN_COUNT];    // chunks per page sizes
-} mi_stats_t;
-
-#undef MI_STAT_COUNT
-#undef MI_STAT_COUNTER
-
-// Initialization
-static inline void mi_stats_header_init(mi_stats_t* stats) {
-  stats->size = sizeof(*stats);
-  stats->version = MI_STAT_VERSION;
-}
-static inline void mi_stats_init(mi_stats_t* stats) {
-  memset(stats,0,sizeof(*stats));
-  mi_stats_header_init(stats);
-}
-
-#define mi_stats_t_decl(name)  mi_stats_t name; mi_stats_init(&name);
-
-// Exported definitions
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// stats from a heap
-mi_decl_export bool    mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept;
-mi_decl_export char*   mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept;      // use mi_free to free the result if the input buf == NULL
-mi_decl_export void    mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept;
-
-// stats from a subprocess and its heaps aggregated
-mi_decl_export bool    mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;
-mi_decl_export char*   mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept;      // use mi_free to free the result if the input buf == NULL
-mi_decl_export void    mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;
-// print subprocess and all its heap stats segregated
-mi_decl_export void    mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;
-
-// stats aggregated for the current subprocess and all its heaps.
-mi_decl_export bool    mi_stats_get(mi_stats_t* stats) mi_attr_noexcept;
-mi_decl_export char*   mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept;      // use mi_free to free the result if the input buf == NULL
-mi_decl_export void    mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
-
-// add the stats of the heap to the subprocess and clear the heap stats
-mi_decl_export void    mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
-
-// stats from the subprocess without aggregating its heaps
-mi_decl_export bool    mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;
-
-mi_decl_export char*   mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept;      // use mi_free to free the result if the input buf == NULL
-mi_decl_export size_t  mi_stats_get_bin_size(size_t bin) mi_attr_noexcept;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif // MIMALLOC_STATS_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc.h
deleted file mode 100644
index f373f5dc..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc.h
+++ /dev/null
@@ -1,706 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MIMALLOC_H
-#define MIMALLOC_H
-
-#define MI_MALLOC_VERSION 30302   // major + 2 digits minor + 2 digits patch
-
-// ------------------------------------------------------
-// Compiler specific attributes
-// ------------------------------------------------------
-
-#ifdef __cplusplus
-  #if (__cplusplus >= 201103L) || (_MSC_VER > 1900)  // C++11
-    #define mi_attr_noexcept   noexcept
-  #else
-    #define mi_attr_noexcept   throw()
-  #endif
-#else
-  #define mi_attr_noexcept
-#endif
-
-#if defined(__cplusplus) && (__cplusplus >= 201703)
-  #define mi_decl_nodiscard    [[nodiscard]]
-#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)  // includes clang, icc, and clang-cl
-  #define mi_decl_nodiscard    __attribute__((warn_unused_result))
-#elif defined(_HAS_NODISCARD)
-  #define mi_decl_nodiscard    _NODISCARD
-#elif (_MSC_VER >= 1700)
-  #define mi_decl_nodiscard    _Check_return_
-#else
-  #define mi_decl_nodiscard
-#endif
-
-#if defined(_MSC_VER) || defined(__MINGW32__)
-  #if !defined(MI_SHARED_LIB)
-    #define mi_decl_export
-  #elif defined(MI_SHARED_LIB_EXPORT)
-    #define mi_decl_export              __declspec(dllexport)
-  #else
-    #define mi_decl_export              __declspec(dllimport)
-  #endif
-  #if defined(__MINGW32__)
-    #define mi_decl_restrict
-    #define mi_attr_malloc              __attribute__((malloc))
-  #else
-    #if (_MSC_VER >= 1900) && !defined(__EDG__)
-      #define mi_decl_restrict          __declspec(allocator) __declspec(restrict)
-    #else
-      #define mi_decl_restrict          __declspec(restrict)
-    #endif
-    #define mi_attr_malloc
-  #endif
-  #define mi_cdecl                      __cdecl
-  #define mi_attr_alloc_size(s)
-  #define mi_attr_alloc_size2(s1,s2)
-  #define mi_attr_alloc_align(p)
-#elif defined(__GNUC__)                 // includes clang and icc
-  #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT)
-    #define mi_decl_export              __attribute__((visibility("default")))
-  #else
-    #define mi_decl_export
-  #endif
-  #define mi_cdecl                      // leads to warnings... __attribute__((cdecl))
-  #define mi_decl_restrict
-  #define mi_attr_malloc                __attribute__((malloc))
-  #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)
-    #define mi_attr_alloc_size(s)
-    #define mi_attr_alloc_size2(s1,s2)
-    #define mi_attr_alloc_align(p)
-  #elif defined(__INTEL_COMPILER)
-    #define mi_attr_alloc_size(s)       __attribute__((alloc_size(s)))
-    #define mi_attr_alloc_size2(s1,s2)  __attribute__((alloc_size(s1,s2)))
-    #define mi_attr_alloc_align(p)
-  #else
-    #define mi_attr_alloc_size(s)       __attribute__((alloc_size(s)))
-    #define mi_attr_alloc_size2(s1,s2)  __attribute__((alloc_size(s1,s2)))
-    #define mi_attr_alloc_align(p)      __attribute__((alloc_align(p)))
-  #endif
-#else
-  #define mi_cdecl
-  #define mi_decl_export
-  #define mi_decl_restrict
-  #define mi_attr_malloc
-  #define mi_attr_alloc_size(s)
-  #define mi_attr_alloc_size2(s1,s2)
-  #define mi_attr_alloc_align(p)
-#endif
-
-// ------------------------------------------------------
-// Includes
-// ------------------------------------------------------
-
-#include <stddef.h>     // size_t
-#include <stdbool.h>    // bool
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// ------------------------------------------------------
-// Standard malloc interface
-// ------------------------------------------------------
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
-mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize)      mi_attr_noexcept mi_attr_alloc_size(2);
-mi_decl_export void* mi_expand(void* p, size_t newsize)                         mi_attr_noexcept mi_attr_alloc_size(2);
-
-mi_decl_export void mi_free(void* p) mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc;
-mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
-mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept;
-
-// ------------------------------------------------------
-// Extended allocation functions
-// ------------------------------------------------------
-#define MI_SMALL_WSIZE_MAX  (128)
-#define MI_SMALL_SIZE_MAX   (MI_SMALL_WSIZE_MAX*sizeof(void*))
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size)       mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
-mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size)        mi_attr_noexcept mi_attr_alloc_size2(2,3);
-mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize)                   mi_attr_noexcept mi_attr_alloc_size(2);
-
-mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size)     mi_attr_noexcept;
-
-// `mi_free_small` is for special applications like language runtimes.
-// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free`
-mi_decl_export void mi_free_small(void* p) mi_attr_noexcept;  
-
-// -------------------------------------------------------------------------------------
-// Aligned allocation
-// Note that `alignment` always follows `size` for consistency with unaligned
-// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`.
-// -------------------------------------------------------------------------------------
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2) mi_attr_alloc_align(3);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2);
-mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
-mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
-
-
-// ------------------------------------------------------
-// Typed allocation, the type is always the first parameter
-// ------------------------------------------------------
-
-#define mi_malloc_tp(tp)                 ((tp*)mi_malloc(sizeof(tp)))
-#define mi_zalloc_tp(tp)                 ((tp*)mi_zalloc(sizeof(tp)))
-#define mi_calloc_tp(tp,n)               ((tp*)mi_calloc(n,sizeof(tp)))
-#define mi_mallocn_tp(tp,n)              ((tp*)mi_mallocn(n,sizeof(tp)))
-#define mi_reallocn_tp(tp,p,n)           ((tp*)mi_reallocn(p,n,sizeof(tp)))
-#define mi_recalloc_tp(tp,p,n)           ((tp*)mi_recalloc(p,n,sizeof(tp)))
-
-#define mi_heap_malloc_tp(tp,hp)         ((tp*)mi_heap_malloc(hp,sizeof(tp)))
-#define mi_heap_zalloc_tp(tp,hp)         ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
-#define mi_heap_calloc_tp(tp,hp,n)       ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))
-#define mi_heap_mallocn_tp(tp,hp,n)      ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))
-#define mi_heap_reallocn_tp(tp,hp,p,n)   ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp)))
-#define mi_heap_recalloc_tp(tp,hp,p,n)   ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp)))
-
-
-// ------------------------------------------------------
-// Internals
-// See also `mimalloc-stats.h` for statistics
-// ------------------------------------------------------
-
-typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
-mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept;
-
-typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg);
-mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept;
-
-typedef void (mi_cdecl mi_error_fun)(int err, void* arg);
-mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);
-
-mi_decl_export void mi_collect(bool force)      mi_attr_noexcept;
-mi_decl_export int  mi_version(void)            mi_attr_noexcept;
-mi_decl_export void mi_options_print(void)      mi_attr_noexcept;
-mi_decl_export void mi_process_info_print(void) mi_attr_noexcept;
-mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg)      mi_attr_noexcept;
-mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
-mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs,
-                                    size_t* current_rss, size_t* peak_rss,
-                                    size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept;
-
-
-
-// Generally do not use the following as these are usually called automatically
-mi_decl_export void mi_process_init(void)     mi_attr_noexcept;
-mi_decl_export void mi_cdecl mi_process_done(void) mi_attr_noexcept;
-mi_decl_export void mi_thread_init(void)      mi_attr_noexcept;
-mi_decl_export void mi_thread_done(void)      mi_attr_noexcept;
-mi_decl_export void mi_thread_set_in_threadpool(void) mi_attr_noexcept; // communicate that a thread is in a threadpool
-
-
-// -----------------------------------------------------------------
-// Return allocated block size (if the return value is not NULL)
-// -----------------------------------------------------------------
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc(size_t size, size_t* block_size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* block_size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
-mi_decl_nodiscard mi_decl_export void* mi_urealloc(void* p, size_t newsize, size_t* block_size_pre, size_t* block_size_post) mi_attr_noexcept mi_attr_alloc_size(2);
-mi_decl_export void mi_ufree(void* p, size_t* block_size) mi_attr_noexcept;
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-
-
-// -------------------------------------------------------------------------------------
-// Heaps: first-class. Can allocate from any thread (and be free'd from any thread)
-// Heaps keep allocations in separate pages from each other (but share the arena's and free'd pages)
-// -------------------------------------------------------------------------------------
-
-struct mi_heap_s;
-typedef struct mi_heap_s mi_heap_t;
-
-mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void);
-mi_decl_export void mi_heap_delete(mi_heap_t* heap);            // move live blocks to the main heap
-mi_decl_export void mi_heap_destroy(mi_heap_t* heap);           // free all live blocks
-mi_decl_export void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
-mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force);
-
-mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_main(void);
-mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_of(const void* p);
-mi_decl_nodiscard mi_decl_export bool       mi_heap_contains(const mi_heap_t* heap, const void* p);
-mi_decl_nodiscard mi_decl_export bool       mi_any_heap_contains(const void* p);
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-
-mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize)              mi_attr_noexcept mi_attr_alloc_size(3);
-mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size)  mi_attr_noexcept mi_attr_alloc_size2(3, 4);
-mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* theap, void* p, size_t newsize)            mi_attr_noexcept mi_attr_alloc_size(3);
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s)            mi_attr_noexcept mi_attr_malloc;
-mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
-mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept;
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
-mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
-mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
-
-
-// --------------------------------------------------------------------------------
-// Zero initialized re-allocation.
-// Only valid on memory that was originally allocated with zero initialization too.
-// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.
-// see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992>
-// --------------------------------------------------------------------------------
-
-mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize)                mi_attr_noexcept mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size)  mi_attr_noexcept mi_attr_alloc_size2(2,3);
-
-mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
-mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4);
-mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3);
-
-mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize)                mi_attr_noexcept mi_attr_alloc_size(3);
-mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size)  mi_attr_noexcept mi_attr_alloc_size2(3, 4);
-
-mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
-mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
-mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5);
-mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
-
-
-
-// ------------------------------------------------------
-// Visiting pages and individual blocks in a heap.
-// ------------------------------------------------------
-
-// An area of heap space contains blocks of a single size.
-typedef struct mi_heap_area_s {
-  void*  blocks;      // start of the area containing theap blocks
-  size_t reserved;    // bytes reserved for this area (virtual)
-  size_t committed;   // current available bytes for this area
-  size_t used;        // number of allocated blocks
-  size_t block_size;  // size in bytes of each block
-  size_t full_block_size; // size in bytes of a full block including padding and metadata.
-  void*  reserved1;   // internal
-} mi_heap_area_t;
-
-typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
-
-mi_decl_export bool   mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
-mi_decl_export bool   mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
-
-
-// ------------------------------------------------------
-// Arena memory management
-// Arena's are larger memory area's provided by the OS or user
-// ------------------------------------------------------
-
-mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;
-
-mi_decl_export int    mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;
-mi_decl_export int    mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;
-
-mi_decl_export int    mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
-mi_decl_export bool   mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_pinned /* cannot decommit/reset? */, bool is_zero, int numa_node) mi_attr_noexcept;
-
-mi_decl_export void   mi_debug_show_arenas(void) mi_attr_noexcept;
-mi_decl_export void   mi_arenas_print(void) mi_attr_noexcept;
-mi_decl_export size_t mi_arena_min_alignment(void);
-mi_decl_export size_t mi_arena_min_size(void);
-
-typedef void* mi_arena_id_t;
-mi_decl_export void*  mi_arena_area(mi_arena_id_t arena_id, size_t* size);
-mi_decl_export int    mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
-mi_decl_export int    mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
-mi_decl_export bool   mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
-mi_decl_export bool   mi_arena_contains(mi_arena_id_t arena_id, const void* p);
-
-// Create a heap that only allocates in the specified arena
-mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
-
-
-// ------------------------------------------------------
-// Subprocesses
-// Advanced: allow sub-processes whose memory arena's stay fully separated (and no reclamation between them).
-// Used for example for separate interpreters in one process.
-// ------------------------------------------------------
-
-typedef struct { void* _mi_subproc_id; } mi_subproc_id_t;  // abstract type
-mi_decl_export mi_subproc_id_t mi_subproc_main(void);
-mi_decl_export mi_subproc_id_t mi_subproc_current(void);
-mi_decl_export mi_subproc_id_t mi_subproc_new(void);
-mi_decl_export void mi_subproc_destroy(mi_subproc_id_t subproc);
-mi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet)
-
-typedef bool (mi_cdecl mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
-mi_decl_export bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
-
-
-// -------------------------------------------------------------------------------------
-// A "theap" is a thread-local heap. This API is only provided for special circumstances like runtimes
-// that already have a thread-local context and can store the theap there for (slightly) faster allocations.
-// This also allows to set a default theap for the current thread so that `malloc` etc. allocate from
-// that theap (instead of the main (t)heap).
-// Theaps are first-class, but can only allocate from the same thread that created it.
-// Allocation through a `theap` may be a tiny bit faster than using plain malloc
-// (as we don't need to lookup the thread local variable).
-// -------------------------------------------------------------------------------------
-
-struct mi_theap_s;
-typedef struct mi_theap_s mi_theap_t;
-
-mi_decl_export mi_theap_t* mi_heap_theap(mi_heap_t* heap);
-mi_decl_export mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
-mi_decl_export mi_theap_t* mi_theap_get_default(void);
-mi_decl_export void        mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept;
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
-mi_decl_nodiscard mi_decl_export                  void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize)              mi_attr_noexcept mi_attr_alloc_size(3);
-
-
-// ------------------------------------------------------
-// Experimental
-// ------------------------------------------------------
-
-// Experimental: objects followed by a guard page.
-// Setting the sample rate on a specific theap can be used to test parts of the program more
-// specifically (in combination with `mi_theap_set_default`).
-// A sample rate of 0 disables guarded objects, while 1 uses a guard page for every object.
-// A seed of 0 uses a random start point. Only objects within the size bound are eligable for guard pages.
-mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed);
-mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max);
-
-// very experimental
-typedef bool (mi_cdecl mi_commit_fun_t)(bool commit, void* start, size_t size, bool* is_zero, void* user_arg);
-mi_decl_export bool  mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive,
-                                      mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept;
-
-//mi_decl_export bool  mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* size);
-//mi_decl_export bool  mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id);
-//mi_decl_export bool  mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena);
-//mi_decl_export void  mi_theap_unload(mi_theap_t* theap);
-
-// unsafe: assumes the page belonging to `p` is only accessed by the calling thread.
-mi_decl_export bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept;
-
-// ------------------------------------------------------
-// Deprecated
-// ------------------------------------------------------
-
-mi_decl_export bool mi_check_owned(const void* p);
-
-mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
-mi_decl_export bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
-
-mi_decl_export int  mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
-mi_decl_export void mi_collect_reduce(size_t target_thread_owned) mi_attr_noexcept;
-
-mi_decl_export void mi_stats_reset(void)      mi_attr_noexcept;
-mi_decl_export void mi_stats_merge(void)      mi_attr_noexcept;
-mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept;  // backward compatibility: `out` is ignored and should be NULL
-
-mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;  // not deprecated but declared in `mimalloc-stats.h` now.
-
-
-// ------------------------------------------------------
-// Options
-// ------------------------------------------------------
-
-typedef enum mi_option_e {
-  // stable options
-  mi_option_show_errors,                // print error messages
-  mi_option_show_stats,                 // print statistics on termination
-  mi_option_verbose,                    // print verbose messages
-  // advanced options
-  mi_option_deprecated_eager_commit,    
-  mi_option_arena_eager_commit,         // eager commit arenas? Use 2 to enable just on overcommit systems (=2)
-  mi_option_purge_decommits,            // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
-  mi_option_allow_large_os_pages,       // allow use of large (2 or 4 MiB) OS pages, implies eager commit.
-  mi_option_reserve_huge_os_pages,      // reserve N huge OS pages (1GiB pages) at startup
-  mi_option_reserve_huge_os_pages_at,   // reserve huge OS pages at a specific NUMA node
-  mi_option_reserve_os_memory,          // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`)
-  mi_option_deprecated_segment_cache,
-  mi_option_deprecated_page_reset,
-  mi_option_deprecated_abandoned_page_purge,
-  mi_option_deprecated_segment_reset,
-  mi_option_deprecated_eager_commit_delay, 
-  mi_option_purge_delay,                // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10)
-  mi_option_use_numa_nodes,             // 0 = use all available numa nodes, otherwise use at most N nodes.
-  mi_option_disallow_os_alloc,          // 1 = do not use OS memory for allocation (but only programmatically reserved arenas)
-  mi_option_os_tag,                     // tag used for OS logging (macOS only for now) (=100)
-  mi_option_max_errors,                 // issue at most N error messages
-  mi_option_max_warnings,               // issue at most N warning messages
-  mi_option_deprecated_max_segment_reclaim,  // max. percentage of the abandoned segments can be reclaimed per try (=10%)
-  mi_option_destroy_on_exit,            // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
-  mi_option_arena_reserve,              // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`)
-  mi_option_arena_purge_mult,           // multiplier for `purge_delay` for the purging delay for arenas (=10)
-  mi_option_deprecated_purge_extend_delay,
-  mi_option_disallow_arena_alloc,       // 1 = do not use arena's for allocation (except if using specific arena id's)
-  mi_option_retry_on_oom,               // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
-  mi_option_visit_abandoned,            // allow visiting theap blocks from abandoned threads (=0)
-  mi_option_guarded_min,                // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
-  mi_option_guarded_max,                // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
-  mi_option_guarded_precise,            // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
-  mi_option_guarded_sample_rate,        // 1 out of N allocations in the min/max range will be guarded (=1000)
-  mi_option_guarded_sample_seed,        // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
-  mi_option_generic_collect,            // collect theaps every N (=10000) generic allocation calls
-  mi_option_page_reclaim_on_free,       // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always
-  mi_option_page_full_retain,           // retain N full (small) pages per size class (=2)
-  mi_option_page_max_candidates,        // max candidate pages to consider for allocation (=4)
-  mi_option_max_vabits,                 // max user space virtual address bits to consider (=48)
-  mi_option_pagemap_commit,             // commit the full pagemap (to always catch invalid pointer uses) (=0)
-  mi_option_page_commit_on_demand,      // commit page memory on-demand
-  mi_option_page_max_reclaim,           // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited))
-  mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16)
-  mi_option_allow_thp,                  // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.
-  mi_option_minimal_purge_size,         // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled.
-  mi_option_arena_max_object_size,      // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). 
-  mi_option_arena_is_numa_local,        // experimental
-  _mi_option_last,
-  // legacy option names
-  mi_option_large_os_pages = mi_option_allow_large_os_pages,
-  mi_option_eager_region_commit = mi_option_arena_eager_commit,
-  mi_option_reset_decommits = mi_option_purge_decommits,
-  mi_option_reset_delay = mi_option_purge_delay,
-  mi_option_limit_os_alloc = mi_option_disallow_os_alloc
-} mi_option_t;
-
-
-mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);
-mi_decl_export void mi_option_enable(mi_option_t option);
-mi_decl_export void mi_option_disable(mi_option_t option);
-mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
-mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
-
-mi_decl_nodiscard mi_decl_export long   mi_option_get(mi_option_t option);
-mi_decl_nodiscard mi_decl_export long   mi_option_get_clamp(mi_option_t option, long min, long max);
-mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option);
-mi_decl_export void mi_option_set(mi_option_t option, long value);
-mi_decl_export void mi_option_set_default(mi_option_t option, long value);
-
-
-// -------------------------------------------------------------------------------------------------------
-// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.
-// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)
-// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing.
-// -------------------------------------------------------------------------------------------------------
-
-mi_decl_export void  mi_cfree(void* p) mi_attr_noexcept;
-mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p)        mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size)     mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
-
-mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size)   mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
-
-mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
-mi_decl_nodiscard mi_decl_export int   mi_reallocarr(void* ptrp, size_t count, size_t size) mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;
-
-mi_decl_export void mi_free_size(void* p, size_t size)                           mi_attr_noexcept;
-mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;
-mi_decl_export void mi_free_aligned(void* p, size_t alignment)                   mi_attr_noexcept;
-mi_decl_export int  mi_dupenv_s(char** buf, size_t* size, const char* name)      mi_attr_noexcept;
-
-// wide characters
-#include <wchar.h>  // wchar_t
-mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name)       mi_attr_noexcept;
-mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s)  mi_attr_noexcept mi_attr_malloc;
-mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s)  mi_attr_noexcept mi_attr_malloc;
-
-// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
-// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size)                   mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size)           mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size)   mi_attr_malloc mi_attr_alloc_size2(1, 2);
-mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize)                mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
-
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size)                 mi_attr_malloc mi_attr_alloc_size(2);
-mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
-
-#ifdef __cplusplus
-}
-#endif
-
-// ---------------------------------------------------------------------------------------------
-// Implement the C++ std::allocator interface for use in STL containers.
-// (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally)
-// ---------------------------------------------------------------------------------------------
-#ifdef __cplusplus
-
-#include <cstddef>     // std::size_t
-#include <cstdint>     // PTRDIFF_MAX
-#if (__cplusplus >= 201103L) || (_MSC_VER > 1900)  // C++11
-#include <type_traits> // std::true_type
-#include <utility>     // std::forward
-#endif
-
-template<class T> struct _mi_stl_allocator_common {
-  typedef T                 value_type;
-  typedef std::size_t       size_type;
-  typedef std::ptrdiff_t    difference_type;
-  typedef value_type&       reference;
-  typedef value_type const& const_reference;
-  typedef value_type*       pointer;
-  typedef value_type const* const_pointer;
-
-  #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900))  // C++11
-  using propagate_on_container_copy_assignment = std::true_type;
-  using propagate_on_container_move_assignment = std::true_type;
-  using propagate_on_container_swap            = std::true_type;
-  template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); }
-  template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); }
-  #else
-  void construct(pointer p, value_type const& val) { ::new(p) value_type(val); }
-  void destroy(pointer p) { p->~value_type(); }
-  #endif
-
-  size_type     max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); }
-  pointer       address(reference x) const        { return &x; }
-  const_pointer address(const_reference x) const  { return &x; }
-};
-
-template<class T> struct mi_stl_allocator : public _mi_stl_allocator_common<T> {
-  using typename _mi_stl_allocator_common<T>::size_type;
-  using typename _mi_stl_allocator_common<T>::value_type;
-  using typename _mi_stl_allocator_common<T>::pointer;
-  template <class U> struct rebind { typedef mi_stl_allocator<U> other; };
-
-  mi_stl_allocator()                                             mi_attr_noexcept = default;
-  mi_stl_allocator(const mi_stl_allocator&)                      mi_attr_noexcept = default;
-  template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { }
-  mi_stl_allocator  select_on_container_copy_construction() const { return *this; }
-  void              deallocate(T* p, size_type) { mi_free(p); }
-
-  #if (__cplusplus >= 201703L)  // C++17
-  mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); }
-  mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
-  #else
-  mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); }
-  #endif
-
-  #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900))  // C++11
-  using is_always_equal = std::true_type;
-  #endif
-};
-
-template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; }
-template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; }
-
-
-#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900)  // C++11
-#define MI_HAS_HEAP_STL_ALLOCATOR 1
-
-#include <memory>      // std::shared_ptr
-
-// Common base class for STL allocators in a specific theap
-template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> {
-  using typename _mi_stl_allocator_common<T>::size_type;
-  using typename _mi_stl_allocator_common<T>::value_type;
-  using typename _mi_stl_allocator_common<T>::pointer;
-
-  _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp, [](mi_heap_t*) {}) {}    /* will not delete nor destroy the passed in heap */
-
-  #if (__cplusplus >= 201703L)  // C++17
-  mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); }
-  mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
-  #else
-  mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); }
-  #endif
-
-  #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900))  // C++11
-  using is_always_equal = std::false_type;
-  #endif
-
-  void collect(bool force) { mi_heap_collect(this->heap.get(), force); }
-  template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); }
-
-protected:
-  std::shared_ptr<mi_heap_t> heap;
-  template<class U, bool D> friend struct _mi_heap_stl_allocator_common;
-
-  _mi_heap_stl_allocator_common() {
-    mi_heap_t* hp = mi_heap_new();
-    this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete));  /* calls heap_delete/destroy when the refcount drops to zero */
-  }
-  _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }
-  template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { }
-
-private:
-  static void heap_delete(mi_heap_t* hp)  { if (hp != NULL) { mi_heap_delete(hp); } }
-  static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } }
-};
-
-// STL allocator allocation in a specific heap
-template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> {
-  using typename _mi_heap_stl_allocator_common<T, false>::size_type;
-  mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called
-  mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { }  // no delete nor destroy on the passed in heap
-  template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { }
-
-  mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; }
-  void deallocate(T* p, size_type) { mi_free(p); }
-  template<class U> struct rebind { typedef mi_heap_stl_allocator<U> other; };
-};
-
-template<class T1, class T2> bool operator==(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
-template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
-
-
-// STL allocator allocation in a specific heap, where `free` does nothing and
-// the heap is destroyed in one go on destruction -- use with care!
-template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> {
-  using typename _mi_heap_stl_allocator_common<T, true>::size_type;
-  mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called
-  mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { }  // no delete nor destroy on the passed in heap
-  template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { }
-
-  mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; }
-  void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ }
-  template<class U> struct rebind { typedef mi_heap_destroy_stl_allocator<U> other; };
-};
-
-template<class T1, class T2> bool operator==(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
-template<class T1, class T2> bool operator!=(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
-
-#endif // C++11
-
-#endif // __cplusplus
-
-#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/atomic.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/atomic.h
deleted file mode 100644
index e4abde6b..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/atomic.h
+++ /dev/null
@@ -1,562 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2024 Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MI_ATOMIC_H
-#define MI_ATOMIC_H
-
-// include windows.h or pthreads.h
-#if defined(_WIN32)
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif
-#include <windows.h>
-#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))
-#define  MI_USE_PTHREADS
-#include <pthread.h>
-#endif
-
-// --------------------------------------------------------------------------------------------
-// Atomics
-// We need to be portable between C, C++, and MSVC.
-// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode.
-// This is why we try to use only `uintptr_t` and `<type>*` as atomic types.
-// To gain better insight in the range of used atomics, we use explicitly named memory order operations
-// instead of passing the memory order as a parameter.
-// -----------------------------------------------------------------------------------------------
-
-#if defined(__cplusplus)
-// Use C++ atomics
-#include <atomic>
-#define  _Atomic(tp)              std::atomic<tp>
-#define  mi_atomic(name)          std::atomic_##name
-#define  mi_memory_order(name)    std::memory_order_##name
-#if (__cplusplus >= 202002L)      // c++20, see issue #571
- #define MI_ATOMIC_VAR_INIT(x)    x
-#elif !defined(ATOMIC_VAR_INIT)
- #define MI_ATOMIC_VAR_INIT(x)    x
-#else
- #define MI_ATOMIC_VAR_INIT(x)    ATOMIC_VAR_INIT(x)
-#endif
-#elif defined(_MSC_VER)
-// Use MSVC C wrapper for C11 atomics
-#define  _Atomic(tp)              tp
-#define  MI_ATOMIC_VAR_INIT(x)    x
-#define  mi_atomic(name)          mi_atomic_##name
-#define  mi_memory_order(name)    mi_memory_order_##name
-#else
-// Use C11 atomics
-#include <stdatomic.h>
-#define  mi_atomic(name)          atomic_##name
-#define  mi_memory_order(name)    memory_order_##name
-#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735
- #define MI_ATOMIC_VAR_INIT(x)    x
-#elif !defined(ATOMIC_VAR_INIT)
- #define MI_ATOMIC_VAR_INIT(x)    x
-#else
- #define MI_ATOMIC_VAR_INIT(x)    ATOMIC_VAR_INIT(x)
-#endif
-#endif
-
-// Various defines for all used memory orders in mimalloc
-#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail)  \
-  mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail)
-
-#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail)  \
-  mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail)
-
-#define mi_atomic_load_acquire(p)                mi_atomic(load_explicit)(p,mi_memory_order(acquire))
-#define mi_atomic_load_relaxed(p)                mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
-#define mi_atomic_store_release(p,x)             mi_atomic(store_explicit)(p,x,mi_memory_order(release))
-#define mi_atomic_store_relaxed(p,x)             mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
-#define mi_atomic_exchange_relaxed(p,x)          mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed))
-#define mi_atomic_exchange_release(p,x)          mi_atomic(exchange_explicit)(p,x,mi_memory_order(release))
-#define mi_atomic_exchange_acq_rel(p,x)          mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel))
-
-#define mi_atomic_cas_weak_relaxed(p,exp,des)    mi_atomic_cas_weak(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))
-#define mi_atomic_cas_weak_release(p,exp,des)    mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
-#define mi_atomic_cas_weak_acq_rel(p,exp,des)    mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
-#define mi_atomic_cas_strong_relaxed(p,exp,des)  mi_atomic_cas_strong(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))
-#define mi_atomic_cas_strong_release(p,exp,des)  mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
-#define mi_atomic_cas_strong_acq_rel(p,exp,des)  mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))
-
-#define mi_atomic_add_relaxed(p,x)               mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed))
-#define mi_atomic_add_acq_rel(p,x)               mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel))
-#define mi_atomic_sub_relaxed(p,x)               mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed))
-#define mi_atomic_sub_acq_rel(p,x)               mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel))
-#define mi_atomic_and_relaxed(p,x)               mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(relaxed))
-#define mi_atomic_and_acq_rel(p,x)               mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel))
-#define mi_atomic_or_relaxed(p,x)                mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(relaxed))
-#define mi_atomic_or_acq_rel(p,x)                mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel))
-
-#define mi_atomic_increment_relaxed(p)           mi_atomic_add_relaxed(p,(uintptr_t)1)
-#define mi_atomic_decrement_relaxed(p)           mi_atomic_sub_relaxed(p,(uintptr_t)1)
-#define mi_atomic_increment_acq_rel(p)           mi_atomic_add_acq_rel(p,(uintptr_t)1)
-#define mi_atomic_decrement_acq_rel(p)           mi_atomic_sub_acq_rel(p,(uintptr_t)1)
-
-static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add);
-static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);
-
-
-#if defined(__cplusplus) || !defined(_MSC_VER)
-
-// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value)
-// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well
-#define mi_atomic_load_ptr_acquire(tp,p)                mi_atomic_load_acquire(p)
-#define mi_atomic_load_ptr_relaxed(tp,p)                mi_atomic_load_relaxed(p)
-
-// In C++ we need to add casts to help resolve templates if NULL is passed
-#if defined(__cplusplus)
-#define mi_atomic_store_ptr_release(tp,p,x)             mi_atomic_store_release(p,(tp*)x)
-#define mi_atomic_store_ptr_relaxed(tp,p,x)             mi_atomic_store_relaxed(p,(tp*)x)
-#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des)    mi_atomic_cas_weak_release(p,exp,(tp*)des)
-#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des)    mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des)
-#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des)  mi_atomic_cas_strong_release(p,exp,(tp*)des)
-#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des)  mi_atomic_cas_strong_acq_rel(p,exp,(tp*)des)
-#define mi_atomic_exchange_ptr_relaxed(tp,p,x)          mi_atomic_exchange_relaxed(p,(tp*)x)
-#define mi_atomic_exchange_ptr_release(tp,p,x)          mi_atomic_exchange_release(p,(tp*)x)
-#define mi_atomic_exchange_ptr_acq_rel(tp,p,x)          mi_atomic_exchange_acq_rel(p,(tp*)x)
-#else
-#define mi_atomic_store_ptr_release(tp,p,x)             mi_atomic_store_release(p,x)
-#define mi_atomic_store_ptr_relaxed(tp,p,x)             mi_atomic_store_relaxed(p,x)
-#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des)    mi_atomic_cas_weak_release(p,exp,des)
-#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des)    mi_atomic_cas_weak_acq_rel(p,exp,des)
-#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des)  mi_atomic_cas_strong_release(p,exp,des)
-#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des)  mi_atomic_cas_strong_acq_rel(p,exp,des)
-#define mi_atomic_exchange_ptr_relaxed(tp,p,x)          mi_atomic_exchange_relaxed(p,x)
-#define mi_atomic_exchange_ptr_release(tp,p,x)          mi_atomic_exchange_release(p,x)
-#define mi_atomic_exchange_ptr_acq_rel(tp,p,x)          mi_atomic_exchange_acq_rel(p,x)
-#endif
-
-// These are used by the statistics
-static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) {
-  return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
-}
-static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
-  const int64_t add = mi_atomic_load_relaxed((_Atomic(int64_t)*)padd);
-  if (add != 0) {
-    mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));
-  }
-}
-static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
-  int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p);
-  while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, &current, x)) { /* nothing */ };
-}
-
-// Used by timers
-#define mi_atomic_loadi64_acquire(p)            mi_atomic(load_explicit)(p,mi_memory_order(acquire))
-#define mi_atomic_loadi64_relaxed(p)            mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
-#define mi_atomic_storei64_release(p,x)         mi_atomic(store_explicit)(p,x,mi_memory_order(release))
-#define mi_atomic_storei64_relaxed(p,x)         mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
-
-#define mi_atomic_casi64_strong_acq_rel(p,e,d)  mi_atomic_cas_strong_acq_rel(p,e,d)
-#define mi_atomic_addi64_acq_rel(p,i)           mi_atomic_add_acq_rel(p,i)
-
-
-#elif defined(_MSC_VER)
-
-// Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
-// It is recommended to always compile as C++ when using MSVC.
-
-#include <intrin.h>
-#ifdef _WIN64
-typedef LONG64        msc_intptr_t;
-#define MI_MSC_64(f)  f##64
-#define MI_MSC_XX(f)  f##64
-#else
-typedef LONG          msc_intptr_t;
-#define MI_MSC_64(f)  f
-#define MI_MSC_XX(f)  f##32
-#endif
-
-typedef enum mi_memory_order_e {
-  mi_memory_order_relaxed,
-  mi_memory_order_consume,
-  mi_memory_order_acquire,
-  mi_memory_order_release,
-  mi_memory_order_acq_rel,
-  mi_memory_order_seq_cst
-} mi_memory_order;
-
-static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) {
-  (void)(mo);
-  return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);
-}
-static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) {
-  (void)(mo);
-  return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub));
-}
-static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
-  (void)(mo);
-  return (uintptr_t)MI_MSC_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
-}
-static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
-  (void)(mo);
-  return (uintptr_t)MI_MSC_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
-}
-static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
-  (void)(mo1); (void)(mo2);
-  const uintptr_t read = (uintptr_t)MI_MSC_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected));
-  if (read == *expected) {
-    return true;
-  }
-  else {
-    *expected = read;
-    return false;
-  }
-}
-static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
-  return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2);
-}
-static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) {
-  (void)(mo);
-  return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
-}
-static inline void mi_atomic_thread_fence(mi_memory_order mo) {
-  (void)(mo);
-  _Atomic(uintptr_t) x = 0;
-  mi_atomic_exchange_explicit(&x, 1, mo);
-}
-
-static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {
-  (void)(mo);
-  // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
-  #if defined(_M_IX86) || defined(_M_X64)
-    return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
-  #elif defined(_M_ARM) || defined(_M_ARM64)
-    if (mo == mi_memory_order_relaxed) {
-      return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
-    }
-    else if (mo <= mi_memory_order_acquire) {
-      return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p);
-    }
-    else {
-      const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
-      __dmb(15);  // _ARM(64)_BARRIER_SY
-      return u;
-    }
-  #else
-    #warning "define mi_atomic_load_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
-    return MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
-  #endif
-}
-static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
-  (void)(mo);
-  // assert(mo<=mi_memory_order_release); // others are not used by mimalloc
-  #if defined(_M_IX86) || defined(_M_X64)
-    MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
-  #elif defined(_M_ARM) || defined(_M_ARM64)
-    if (mo == mi_memory_order_relaxed) {
-      MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
-    }
-    else if (mo <= mi_memory_order_release) {
-      MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x);
-    }
-    else {
-      mi_atomic_exchange_explicit(p, x, mo);
-    }
-  #else
-    mi_atomic_exchange_explicit(p, x, mo);
-  #endif
-}
-
-static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) {
-  (void)(mo);
-  // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
-  #if defined(_M_IX86) || defined(_M_X64)
-    return __iso_volatile_load64((volatile const int64_t*)p);
-  #elif defined(_M_ARM) || defined(_M_ARM64)
-    if (mo == mi_memory_order_relaxed) {
-      return __iso_volatile_load64((volatile const int64_t*)p);
-    }
-    #if defined(_M_ARM64)
-    else if (mo <= mi_memory_order_acquire) {
-      return __ldar64((volatile const uintptr_t*)p);
-    }
-    #endif
-    else {
-      const int64_t i = __iso_volatile_load64((volatile const int64_t*)p);
-      __dmb(15);  // _ARM(64)_BARRIER_SY
-      return i;
-    }
-  #else
-    #warning "define mi_atomic_loadi64_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
-    return __iso_volatile_load64((volatile const int64_t*)p);
-  #endif
-}
-
-static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) {
-  (void)(mo);
-  // assert(mo<=mi_memory_order_release); // others are not used by mimalloc
-  #if defined(_M_IX86) || defined(_M_X64)
-    __iso_volatile_store64((volatile int64_t*)p,x);
-  #elif defined(_M_ARM) || defined(_M_ARM64)
-    if (mo == mi_memory_order_relaxed) {
-      __iso_volatile_store64((volatile int64_t*)p,x);
-    }
-    #if defined(_M_ARM64)
-    else if (mo == mi_memory_order_release) {
-      __stlr64((volatile uint64_t*)p, (uint64_t)x);
-    }
-    #endif
-    else {
-      InterlockedExchange64(p, x);
-    }
-  #else
-    InterlockedExchange64(p, x);
-  #endif
-}
-
-// These are used by the statistics
-static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) {
-  #ifdef _WIN64
-    return (int64_t)mi_atomic_addi((int64_t*)p, add);
-  #elif defined(_M_ARM)
-    return _InterlockedExchangeAdd64(p, add);
-  #else
-    // x86
-    int64_t current;
-    int64_t sum;
-    do {
-      current = __iso_volatile_load64((volatile const int64_t*)p);
-      sum = current + add;
-    } while (_InterlockedCompareExchange64(p, sum, current) != current);
-    return current;
-  #endif
-}
-
-static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
-  const int64_t add = *padd;
-  if (add != 0) {
-    mi_atomic_addi64_relaxed((volatile _Atomic(int64_t)*)p, add);
-  }
-}
-
-static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) {
-  int64_t current;
-  do {
-    current = *p;
-  } while (current < x && _InterlockedCompareExchange64(p, x, current) != current);
-}
-
-static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {
-  mi_atomic_addi64_relaxed(p, i);
-}
-
-static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {
-  const int64_t read = _InterlockedCompareExchange64(p, des, *exp);
-  if (read == *exp) {
-    return true;
-  }
-  else {
-    *exp = read;
-    return false;
-  }
-}
-
-// The pointer macros cast to `uintptr_t`.
-#define mi_atomic_load_ptr_acquire(tp,p)                (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))
-#define mi_atomic_load_ptr_relaxed(tp,p)                (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))
-#define mi_atomic_store_ptr_release(tp,p,x)             mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
-#define mi_atomic_store_ptr_relaxed(tp,p,x)             mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
-#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des)    mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
-#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des)    mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
-#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des)  mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
-#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des)  mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
-#define mi_atomic_exchange_ptr_relaxed(tp,p,x)          (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
-#define mi_atomic_exchange_ptr_release(tp,p,x)          (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
-#define mi_atomic_exchange_ptr_acq_rel(tp,p,x)          (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
-
-#define mi_atomic_loadi64_acquire(p)    mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire))
-#define mi_atomic_loadi64_relaxed(p)    mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed))
-#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release))
-#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed))
-
-
-#endif
-
-
-// Atomically add a signed value; returns the previous value.
-static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) {
-  return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add);
-}
-
-// Atomically subtract a signed value; returns the previous value.
-static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {
-  return (intptr_t)mi_atomic_addi(p, -sub);
-}
-
-
-// ----------------------------------------------------------------------
-// Guard
-// ----------------------------------------------------------------------
-
-typedef _Atomic(uintptr_t) mi_atomic_guard_t;
-
-// Allows only one thread to execute at a time (without blocking anyone)
-#define mi_atomic_guard(guard) \
-  uintptr_t _mi_guard_expected = 0; \
-  for(bool _mi_guard_once = true; \
-      _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \
-      (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )
-
-
-// ----------------------------------------------------------------------
-// Locks
-// These should be light-weight in-process only locks.
-// Only used for reserving arena's and to maintain the abandoned list.
-// ----------------------------------------------------------------------
-#if _MSC_VER
-#pragma warning(disable:26110)  // unlock with holding lock
-#endif
-
-#define mi_lock(lock)                  for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) )
-#define mi_lock_maybe(lock,acquire)    for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )
-
-
-#if defined(_WIN32)
-
-typedef struct mi_lock_s {
-  SRWLOCK mutex;    // slim reader-writer lock
-} mi_lock_t;
-
-#define MI_LOCK_INITIALIZER   { SRWLOCK_INIT }
-
-static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
-  return TryAcquireSRWLockExclusive(&lock->mutex);
-}
-static inline void mi_lock_acquire(mi_lock_t* lock) {
-  AcquireSRWLockExclusive(&lock->mutex);
-}
-static inline void mi_lock_release(mi_lock_t* lock) {
-  ReleaseSRWLockExclusive(&lock->mutex);
-}
-static inline void mi_lock_init(mi_lock_t* lock) {
-  InitializeSRWLock(&lock->mutex);
-}
-static inline void mi_lock_done(mi_lock_t* lock) {
-  (void)(lock);
-}
-
-#elif defined(MI_USE_PTHREADS)
-
-#include <string.h> // memcpy
-void _mi_error_message(int err, const char* fmt, ...);
-
-typedef struct mi_lock_s {
-  pthread_mutex_t mutex;
-} mi_lock_t;
-
-#define MI_LOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
-
-static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
-  return (pthread_mutex_trylock(&lock->mutex) == 0);
-}
-static inline void mi_lock_acquire(mi_lock_t* lock) {
-  const int err = pthread_mutex_lock(&lock->mutex);
-  if (err != 0) {
-    _mi_error_message(err, "internal error: lock cannot be acquired (err %i)\n", err);
-  }
-}
-static inline void mi_lock_release(mi_lock_t* lock) {
-  pthread_mutex_unlock(&lock->mutex);
-}
-static inline void mi_lock_init(mi_lock_t* lock) {
-  if(lock==NULL) return;
-  // use this instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)
-  const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
-  memcpy(&lock->mutex,&mutex,sizeof(mutex));
-}
-static inline void mi_lock_done(mi_lock_t* lock) {
-  pthread_mutex_destroy(&lock->mutex);
-}
-
-#elif defined(__cplusplus)
-
-#include <thread>
-#include <mutex>
-#include <new>
-
-typedef struct mi_lock_s {
-  std::mutex mutex;
-} mi_lock_t;
-
-#define MI_LOCK_INITIALIZER   { }
-
-static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
-  return lock->mutex.try_lock();
-}
-static inline void mi_lock_acquire(mi_lock_t* lock) {
-  lock->mutex.lock();
-}
-static inline void mi_lock_release(mi_lock_t* lock) {
-  lock->mutex.unlock();
-}
-static inline void mi_lock_init(mi_lock_t* lock) {
-  new(&lock->mutex) std::mutex();
-}
-static inline void mi_lock_done(mi_lock_t* lock) {
-  (void)(lock);
-}
-
-#else
-
-// fall back to poor man's locks.
-// this should only be the case in a single-threaded environment (like __wasi__)
-#include <errno.h>
-#ifndef EFAULT
-#define EFAULT (14)
-#endif
-void _mi_error_message(int err, const char* fmt, ...);
-void _mi_prim_thread_yield(void);
-
-typedef struct mi_lock_s {
-  _Atomic(uintptr_t) mutex;
-} mi_lock_t;
-
-#define MI_LOCK_INITIALIZER  { MI_ATOMIC_VAR_INIT(0) }
-
-static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
-  uintptr_t expected = 0;
-  return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);
-}
-static inline void mi_lock_acquire(mi_lock_t* lock) {
-  for (int i = 0; i < 10000; i++) {  // for at most 10000 tries?
-    if (mi_lock_try_acquire(lock)) return;
-    _mi_prim_thread_yield();
-  }
-  _mi_error_message(EFAULT, "internal error: lock cannot be acquired (due to lack of native lock primitives)\n");
-}
-static inline void mi_lock_release(mi_lock_t* lock) {
-  mi_atomic_store_release(&lock->mutex, (uintptr_t)0);
-}
-static inline void mi_lock_init(mi_lock_t* lock) {
-  mi_lock_release(lock);
-}
-static inline void mi_lock_done(mi_lock_t* lock) {
-  (void)(lock);
-}
-
-#endif
-
-
-typedef struct mi_atomic_once_s {
-  _Atomic(uintptr_t) tid;
-  mi_lock_t          lock;
-} mi_atomic_once_t;
-
-// Returns `true` only on the first invocation, signifying we can execute an action once.
-// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
-// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
-bool _mi_atomic_once_enter(mi_atomic_once_t* once);        // defined in `libc.c`
-void _mi_atomic_once_release(mi_atomic_once_t* once);      // defined in `libc.c`
-
-#define mi_atomic_do_once  \
-  static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \
-  for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false))
-
-
-#endif // __MIMALLOC_ATOMIC_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/bits.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/bits.h
deleted file mode 100644
index 61787728..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/bits.h
+++ /dev/null
@@ -1,340 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* ----------------------------------------------------------------------------
-  Bit operation, and platform dependent definition (MI_INTPTR_SIZE etc)
----------------------------------------------------------------------------- */
-
-#pragma once
-#ifndef MI_BITS_H
-#define MI_BITS_H
-
-#include <stddef.h>   // size_t
-#include <stdint.h>   // int64_t etc
-#include <stdbool.h>  // bool
-
-// ------------------------------------------------------
-// Size of a pointer.
-// We assume that `sizeof(void*)==sizeof(intptr_t)`
-// and it holds for all platforms we know of.
-//
-// However, the C standard only requires that:
-//  p == (void*)((intptr_t)p))
-// but we also need:
-//  i == (intptr_t)((void*)i)
-// or otherwise one might define an intptr_t type that is larger than a pointer...
-// ------------------------------------------------------
-
-#if INTPTR_MAX > INT64_MAX
-# define MI_INTPTR_SHIFT (4)  // assume 128-bit  (as on arm CHERI for example)
-#elif INTPTR_MAX == INT64_MAX
-# define MI_INTPTR_SHIFT (3)
-#elif INTPTR_MAX == INT32_MAX
-# define MI_INTPTR_SHIFT (2)
-#else
-#error platform pointers must be 32, 64, or 128 bits
-#endif
-
-#if (INTPTR_MAX) > LONG_MAX
-# define MI_PU(x)  x##ULL
-#else
-# define MI_PU(x)  x##UL
-#endif
-
-#if SIZE_MAX == UINT64_MAX
-# define MI_SIZE_SHIFT (3)
-typedef int64_t  mi_ssize_t;
-#elif SIZE_MAX == UINT32_MAX
-# define MI_SIZE_SHIFT (2)
-typedef int32_t  mi_ssize_t;
-#else
-#error platform objects must be 32 or 64 bits in size
-#endif
-
-#if (SIZE_MAX/2) > LONG_MAX
-# define MI_ZU(x)  x##ULL
-#else
-# define MI_ZU(x)  x##UL
-#endif
-
-#define MI_INTPTR_SIZE  (1<<MI_INTPTR_SHIFT)
-#define MI_INTPTR_BITS  (MI_INTPTR_SIZE*8)
-
-#define MI_SIZE_SIZE  (1<<MI_SIZE_SHIFT)
-#define MI_SIZE_BITS  (MI_SIZE_SIZE*8)
-
-#define MI_KiB     (MI_ZU(1024))
-#define MI_MiB     (MI_KiB*MI_KiB)
-#define MI_GiB     (MI_MiB*MI_KiB)
-
-
-/* --------------------------------------------------------------------------------
-  Architecture
--------------------------------------------------------------------------------- */
-
-#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_HYBRID_X86_ARM64) || defined(_M_ARM64EC)  // consider arm64ec as arm64
-#define MI_ARCH_ARM64     1
-#elif defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64)
-#define MI_ARCH_X64       1
-#elif defined(__i386__) || defined(__i386) || defined(_M_IX86) || defined(_X86_) || defined(__X86__)
-#define MI_ARCH_X86       1
-#elif defined(__arm__) || defined(_ARM) || defined(_M_ARM)  || defined(_M_ARMT) || defined(__arm)
-#define MI_ARCH_ARM32     1
-#elif defined(__riscv) || defined(_M_RISCV)
-#define MI_ARCH_RISCV     1
-#if (LONG_MAX == INT32_MAX)
-#define MI_ARCH_RISCV32   1
-#else
-#define MI_ARCH_RISCV64   1
-#endif
-#endif
-
-#if MI_ARCH_X64 && defined(__AVX2__)
-#include <immintrin.h>
-#elif MI_ARCH_ARM64 && MI_OPT_SIMD
-#include <arm_neon.h>
-#endif
-#if defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
-#include <intrin.h>
-#endif
-
-#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // msvc
-#define __BMI2__  1
-#endif
-#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc
-#define __BMI1__  1
-#endif
-#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc
-#define __LZCNT__  1
-#endif
-
-// Define big endian if needed
-// #define MI_BIG_ENDIAN  1
-
-// maximum virtual address bits in a user-space pointer
-#if MI_DEFAULT_VIRTUAL_ADDRESS_BITS > 0 
-#define MI_MAX_VABITS     MI_DEFAULT_VIRTUAL_ADDRESS_BITS
-#elif   MI_ARCH_X64
-#define MI_MAX_VABITS     (47)
-#elif MI_INTPTR_SIZE > 4
-#define MI_MAX_VABITS     (48)
-#else
-#define MI_MAX_VABITS     (32)
-#endif
-
-// use a flat page-map or a 2-level one
-#ifndef MI_PAGE_MAP_FLAT
-#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED
-#define MI_PAGE_MAP_FLAT  1
-#else
-#define MI_PAGE_MAP_FLAT  0
-#endif
-#endif
-
-
-/* --------------------------------------------------------------------------------
-  Builtin's
--------------------------------------------------------------------------------- */
-
-#ifndef __has_builtin
-#define __has_builtin(x)  0
-#endif
-
-#define mi_builtin(name)        __builtin_##name
-#define mi_has_builtin(name)    __has_builtin(__builtin_##name)
-
-#if (LONG_MAX == INT32_MAX)
-#define mi_builtin32(name)       mi_builtin(name##l)
-#define mi_has_builtin32(name)   mi_has_builtin(name##l)
-#else
-#define mi_builtin32(name)       mi_builtin(name)
-#define mi_has_builtin32(name)   mi_has_builtin(name)
-#endif
-#if (LONG_MAX == INT64_MAX)
-#define mi_builtin64(name)       mi_builtin(name##l)
-#define mi_has_builtin64(name)   mi_has_builtin(name##l)
-#else
-#define mi_builtin64(name)       mi_builtin(name##ll)
-#define mi_has_builtin64(name)   mi_has_builtin(name##ll)
-#endif
-
-#if (MI_SIZE_BITS == 32)
-#define mi_builtinz(name)        mi_builtin32(name)
-#define mi_has_builtinz(name)    mi_has_builtin32(name)
-#define mi_msc_builtinz(name)    name
-#elif (MI_SIZE_BITS == 64)
-#define mi_builtinz(name)        mi_builtin64(name)
-#define mi_has_builtinz(name)    mi_has_builtin64(name)
-#define mi_msc_builtinz(name)    name##64
-#endif
-
-/* --------------------------------------------------------------------------------
-  Popcount and count trailing/leading zero's
--------------------------------------------------------------------------------- */
-
-size_t _mi_popcount_generic(size_t x);
-
-static inline size_t mi_popcount(size_t x) {
-  #if mi_has_builtinz(popcount)
-    return mi_builtinz(popcount)(x);
-  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
-    return mi_msc_builtinz(__popcnt)(x);
-  #elif MI_ARCH_X64 && defined(__BMI1__)
-    return (size_t)_mm_popcnt_u64(x);
-  #else
-    #define MI_HAS_FAST_POPCOUNT  0
-    return (x<=1 ? x : _mi_popcount_generic(x));
-  #endif
-}
-
-#ifndef MI_HAS_FAST_POPCOUNT
-#define MI_HAS_FAST_POPCOUNT 1
-#endif
-
-
-
-size_t _mi_clz_generic(size_t x);
-size_t _mi_ctz_generic(size_t x);
-
-static inline size_t mi_ctz(size_t x) {
-  #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) // on x64 tzcnt is defined for 0
-    size_t r;
-    __asm ("tzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
-    return r;
-  #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__) 
-    return _tzcnt_u64(x);
-  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
-    unsigned long idx;
-    return (mi_msc_builtinz(_BitScanForward)(&idx, x) ? (size_t)idx : MI_SIZE_BITS);
-  #elif mi_has_builtinz(ctz)
-    return (x!=0 ? (size_t)mi_builtinz(ctz)(x) : MI_SIZE_BITS);
-  #elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)
-    size_t r = MI_SIZE_BITS;  // bsf leaves destination unmodified if the argument is 0 (see <https://github.com/llvm/llvm-project/pull/102885>)
-    __asm ("bsf\t%1, %0" : "+r"(r) : "r"(x) : "cc");
-    return r;
-  #elif MI_HAS_FAST_POPCOUNT
-    return (x!=0 ? (mi_popcount(x^(x-1))-1) : MI_SIZE_BITS);
-  #else
-    #define MI_HAS_FAST_BITSCAN  0
-    return (x!=0 ? _mi_ctz_generic(x) : MI_SIZE_BITS);
-  #endif
-}
-
-static inline size_t mi_clz(size_t x) {
-  #if defined(__GNUC__) && MI_ARCH_X64 && defined(__LZCNT__) // on x64 lzcnt is defined for 0
-    size_t r;
-    __asm ("lzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
-    return r;
-  #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__) 
-    return _lzcnt_u64(x);
-  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
-    unsigned long idx;
-    return (mi_msc_builtinz(_BitScanReverse)(&idx, x) ? MI_SIZE_BITS - 1 - (size_t)idx : MI_SIZE_BITS);
-  #elif mi_has_builtinz(clz)
-    return (x!=0 ? (size_t)mi_builtinz(clz)(x) : MI_SIZE_BITS);
-  #elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)
-    if (x==0) return MI_SIZE_BITS;
-    size_t r;
-    __asm ("bsr\t%1, %0" : "=r"(r) : "r"(x) : "cc");
-    return (MI_SIZE_BITS - 1 - r);
-  #else
-    #define MI_HAS_FAST_BITSCAN  0
-    return (x!=0 ? _mi_clz_generic(x) : MI_SIZE_BITS);
-  #endif
-}
-
-#ifndef MI_HAS_FAST_BITSCAN
-#define MI_HAS_FAST_BITSCAN 1
-#endif
-
-/* --------------------------------------------------------------------------------
-  find trailing/leading zero  (bit scan forward/reverse)
--------------------------------------------------------------------------------- */
-
-// Bit scan forward: find the least significant bit that is set (i.e. count trailing zero's)
-// return false if `x==0` (with `*idx` undefined) and true otherwise,
-// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
-static inline bool mi_bsf(size_t x, size_t* idx) {
-  #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) && (!defined(__clang_major__) || __clang_major__ >= 9)
-    // on x64 the carry flag is set on zero which gives better codegen
-    bool is_zero;
-    __asm ( "tzcnt\t%2, %1" : "=@ccc"(is_zero), "=r"(*idx) : "r"(x) : "cc" );
-    return !is_zero;
-  #elif 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
-    unsigned long i;
-    return (mi_msc_builtinz(_BitScanForward)(&i, x) ? (*idx = (size_t)i, true) : false);
-  #else
-    return (x!=0 ? (*idx = mi_ctz(x), true) : false);
-  #endif
-}
-
-// Bit scan reverse: find the most significant bit that is set
-// return false if `x==0` (with `*idx` undefined) and true otherwise,
-// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
-static inline bool mi_bsr(size_t x, size_t* idx) {
-  #if 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
-    unsigned long i;
-    return (mi_msc_builtinz(_BitScanReverse)(&i, x) ? (*idx = (size_t)i, true) : false);
-  #else
-    return (x!=0 ? (*idx = MI_SIZE_BITS - 1 - mi_clz(x), true) : false);
-  #endif
-}
-
-
-/* --------------------------------------------------------------------------------
-  rotate
--------------------------------------------------------------------------------- */
-
-static inline size_t mi_rotr(size_t x, size_t r) {
-  #if (mi_has_builtin(rotateright64) && MI_SIZE_BITS==64)
-    return mi_builtin(rotateright64)(x,r);
-  #elif (mi_has_builtin(rotateright32) && MI_SIZE_BITS==32)
-    return mi_builtin(rotateright32)(x,r);
-  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)
-    return _rotr64(x, (int)r);
-  #elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)
-    return _lrotr(x,(int)r);
-  #else
-    // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
-    // avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
-    const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
-    return ((x >> rshift) | (x << ((-rshift) & (MI_SIZE_BITS-1))));
-  #endif
-}
-
-static inline size_t mi_rotl(size_t x, size_t r) {
-  #if (mi_has_builtin(rotateleft64) && MI_SIZE_BITS==64)
-    return mi_builtin(rotateleft64)(x,r);
-  #elif (mi_has_builtin(rotateleft32) && MI_SIZE_BITS==32)
-    return mi_builtin(rotateleft32)(x,r);
-  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)
-    return _rotl64(x, (int)r);
-  #elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)
-    return _lrotl(x, (int)r);
-  #else
-    // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
-    // avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
-    const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
-    return ((x << rshift) | (x >> ((-rshift) & (MI_SIZE_BITS-1))));
-  #endif
-}
-
-static inline uint32_t mi_rotl32(uint32_t x, uint32_t r) {
-  #if mi_has_builtin(rotateleft32)
-    return mi_builtin(rotateleft32)(x,r);
-  #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
-    return _lrotl(x, (int)r);
-  #else
-    // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to
-    // avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
-    const unsigned int rshift = (unsigned int)(r) & 31;
-    return ((x << rshift) | (x >> ((-rshift) & 31)));
-  #endif
-}
-
-
-#endif // MI_BITS_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/internal.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/internal.h
deleted file mode 100644
index febf2559..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/internal.h
+++ /dev/null
@@ -1,1282 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MI_INTERNAL_H
-#define MI_INTERNAL_H
-
-// --------------------------------------------------------------------------
-// This file contains the internal API's of mimalloc and various utility
-// functions and macros.
-// --------------------------------------------------------------------------
-
-#include "types.h"
-#include "track.h"
-#include "bits.h"
-
-
-// --------------------------------------------------------------------------
-// Compiler defines
-// --------------------------------------------------------------------------
-
-#if (MI_DEBUG>0)
-#define mi_trace_message(...)  _mi_trace_message(__VA_ARGS__)
-#else
-#define mi_trace_message(...)
-#endif
-
-#define mi_decl_cache_align     mi_decl_align(64)
-
-#if defined(_MSC_VER)
-#pragma warning(disable:4127)   // suppress constant conditional warning (due to MI_SECURE paths)
-#pragma warning(disable:26812)  // unscoped enum warning
-#define mi_decl_forceinline     __forceinline
-#define mi_decl_noinline        __declspec(noinline)
-#define mi_decl_thread          __declspec(thread)
-#define mi_decl_noreturn        __declspec(noreturn)
-#define mi_decl_weak
-#define mi_decl_hidden
-#define mi_decl_cold
-#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
-#if !MI_TRACK_ASAN
-#define mi_decl_forceinline     __attribute__((always_inline)) inline
-#else
-#define mi_decl_forceinline     inline
-#endif
-#define mi_decl_noinline        __attribute__((noinline))
-#define mi_decl_thread          __thread
-#define mi_decl_noreturn        __attribute__((noreturn))
-#define mi_decl_weak            __attribute__((weak))
-#define mi_decl_hidden          __attribute__((visibility("hidden")))
-#if (__GNUC__ >= 4) || defined(__clang__)
-#define mi_decl_cold            __attribute__((cold))
-#else
-#define mi_decl_cold
-#endif
-#elif __cplusplus >= 201103L    // c++11
-#define mi_decl_forceinline     inline
-#define mi_decl_noinline
-#define mi_decl_thread          thread_local
-#define mi_decl_noreturn        [[noreturn]]
-#define mi_decl_weak
-#define mi_decl_hidden
-#define mi_decl_cold
-#else
-#define mi_decl_forceinline     inline
-#define mi_decl_noinline
-#define mi_decl_thread          __thread        // hope for the best :-)
-#define mi_decl_noreturn
-#define mi_decl_weak
-#define mi_decl_hidden
-#define mi_decl_cold
-#endif
-
-#if defined(__GNUC__) || defined(__clang__)
-#define mi_unlikely(x)     (__builtin_expect(!!(x),false))
-#define mi_likely(x)       (__builtin_expect(!!(x),true))
-#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)
-#define mi_unlikely(x)     (x) [[unlikely]]
-#define mi_likely(x)       (x) [[likely]]
-#else
-#define mi_unlikely(x)     (x)
-#define mi_likely(x)       (x)
-#endif
-
-#ifndef __has_builtin
-#define __has_builtin(x)    0
-#endif
-
-#if defined(__cplusplus)
-#define mi_decl_externc     extern "C"
-#else
-#define mi_decl_externc
-#endif
-
-#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc
-#define mi_decl_maybe_unused    __attribute__((unused))
-#elif __cplusplus >= 201703L    // c++17
-#define mi_decl_maybe_unused    [[maybe_unused]]
-#else
-#define mi_decl_maybe_unused
-#endif
-
-#if defined(__cplusplus)
-#define mi_decl_externc         extern "C"
-#else
-#define mi_decl_externc
-#endif
-
-
-#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
-#define __wasi__
-#endif
-
-
-// --------------------------------------------------------------------------
-// Internal functions
-// --------------------------------------------------------------------------
-
-
-// "libc.c"
-#include <stdarg.h>
-int           _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args);
-int           _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);
-char          _mi_toupper(char c);
-int           _mi_strnicmp(const char* s, const char* t, size_t n);
-void          _mi_strlcpy(char* dest, const char* src, size_t dest_size);
-void          _mi_strlcat(char* dest, const char* src, size_t dest_size);
-size_t        _mi_strlen(const char* s);
-size_t        _mi_strnlen(const char* s, size_t max_len);
-char*         _mi_strnstr(char* s, size_t max_len, const char* pat);
-bool          _mi_streq(const char* s, const char* t);
-bool          _mi_getenv(const char* name, char* result, size_t result_size);
-
-// "options.c"
-void          _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
-void          _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
-void          _mi_raw_message(const char* fmt, ...);
-void          _mi_message(const char* fmt, ...);
-void          _mi_warning_message(const char* fmt, ...);
-void          _mi_verbose_message(const char* fmt, ...);
-void          _mi_trace_message(const char* fmt, ...);
-void          _mi_options_init(void);
-void          _mi_options_post_init(void);
-long          _mi_option_get_fast(mi_option_t option);
-void          _mi_error_message(int err, const char* fmt, ...);
-
-// random.c
-void          _mi_random_init(mi_random_ctx_t* ctx);
-void          _mi_random_init_weak(mi_random_ctx_t* ctx);
-void          _mi_random_reinit_if_weak(mi_random_ctx_t * ctx);
-void          _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);
-uintptr_t     _mi_random_next(mi_random_ctx_t* ctx);
-uintptr_t     _mi_theap_random_next(mi_theap_t* theap);
-uintptr_t     _mi_os_random_weak(uintptr_t extra_seed);
-static inline uintptr_t _mi_random_shuffle(uintptr_t x);
-
-// init.c
-extern mi_decl_hidden mi_decl_cache_align const mi_page_t  _mi_page_empty;
-void          _mi_auto_process_init(void);
-void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept;
-bool          _mi_is_redirected(void);
-bool          _mi_allocator_init(const char** message);
-void          _mi_allocator_done(void);
-bool          _mi_is_main_thread(void);
-bool          _mi_preloading(void);           // true while the C runtime is not initialized yet
-void          _mi_thread_done(mi_theap_t* theap);
-
-mi_subproc_t* _mi_subproc(void);
-mi_subproc_t* _mi_subproc_main(void);
-mi_heap_t*    _mi_subproc_heap_main(mi_subproc_t* subproc);
-mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);
-
-mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
-size_t        _mi_thread_seq_id(void) mi_attr_noexcept;
-bool          _mi_is_heap_main(const mi_heap_t* heap);
-bool          _mi_is_theap_main(const mi_theap_t* theap);
-void          _mi_theap_guarded_init(mi_theap_t* theap);
-void          _mi_theap_options_init(mi_theap_t* theap);
-mi_theap_t*   _mi_theap_default_safe(void);             // ensure the returned theap is initialized
-mi_theap_t*   _mi_theap_main_safe(void);
-   
-// os.c
-void          _mi_os_init(void);                                            // called from process init
-void*         _mi_os_alloc(size_t size, mi_memid_t* memid);
-void*         _mi_os_zalloc(size_t size, mi_memid_t* memid);
-void          _mi_os_free(void* p, size_t size, mi_memid_t memid);
-void          _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc );
-
-size_t        _mi_os_page_size(void);
-size_t        _mi_os_guard_page_size(void);
-size_t        _mi_os_good_alloc_size(size_t size);
-bool          _mi_os_has_overcommit(void);
-bool          _mi_os_has_virtual_reserve(void);
-size_t        _mi_os_virtual_address_bits(void);
-size_t        _mi_os_minimal_purge_size(void);
-
-bool          _mi_os_reset(void* addr, size_t size);
-bool          _mi_os_decommit(void* addr, size_t size);
-void          _mi_os_reuse(void* p, size_t size);
-mi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero);
-mi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size);
-mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size);
-bool          _mi_os_unprotect(void* addr, size_t size);
-bool          _mi_os_purge(void* p, size_t size);
-bool          _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg);
-
-size_t        _mi_os_secure_guard_page_size(void);
-bool          _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid);
-bool          _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid);
-bool          _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid);
-bool          _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid);
-
-int           _mi_os_numa_node(void);
-int           _mi_os_numa_node_count(void);
-
-void*         _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);
-void*         _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);
-
-void*         _mi_os_get_aligned_hint(size_t try_alignment, size_t size);
-bool          _mi_os_canuse_large_page(size_t size, size_t alignment);
-size_t        _mi_os_large_page_size(void);
-void*         _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);
-
-// threadlocal.c
-
-mi_thread_local_t _mi_thread_local_create(void);
-void          _mi_thread_local_free( mi_thread_local_t key );
-bool          _mi_thread_local_set(  mi_thread_local_t key, void* val );
-void*         _mi_thread_local_get(  mi_thread_local_t key );
-void          _mi_thread_locals_init(void);
-void          _mi_thread_locals_done(void);
-void          _mi_thread_locals_thread_done(void);
-
-// arena.c
-mi_arena_id_t _mi_arena_id_none(void);
-mi_arena_t*   _mi_arena_from_id(mi_arena_id_t id);
-bool          _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena);
-
-void*         _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);
-void*         _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);
-void          _mi_arenas_free(void* p, size_t size, mi_memid_t memid);
-bool          _mi_arenas_contain(const void* p);
-void          _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld);
-void          _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc);
-
-mi_page_t*    _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment);
-void          _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);
-void          _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap);
-void          _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);
-bool          _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page);
-
-// arena-meta.c
-void*         _mi_meta_zalloc( size_t size, mi_memid_t* memid );
-void          _mi_meta_free(void* p, size_t size, mi_memid_t memid);
-bool          _mi_meta_is_meta_page(void* p);
-
-// "page-map.c"
-bool          _mi_page_map_init(void);
-mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page);
-void          _mi_page_map_unregister(mi_page_t* page);
-void          _mi_page_map_unregister_range(void* start, size_t size);
-mi_page_t*    _mi_safe_ptr_page(const void* p);
-void          _mi_page_map_unsafe_destroy(mi_subproc_t* subproc);
-
-// "page.c"
-void*         _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable)  mi_attr_noexcept mi_attr_malloc;
-
-void          _mi_page_retire(mi_page_t* page) mi_attr_noexcept;       // free the page if there are no other pages with many free blocks
-void          _mi_page_unfull(mi_page_t* page);
-void          _mi_page_free(mi_page_t* page, mi_page_queue_t* pq);     // free the page
-void          _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq);  // abandon the page, to be picked up by another thread...
-
-size_t        _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append);
-void          _mi_deferred_free(mi_theap_t* theap, bool force);
-
-void          _mi_page_free_collect(mi_page_t* page, bool force);
-void          _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head);
-mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page);
-bool          _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq);
-
-size_t        _mi_page_stats_bin(const mi_page_t* page); // for stats
-size_t        _mi_bin_size(size_t bin);                  // for stats
-size_t        _mi_bin(size_t size);                      // for stats
-
-// "theap.c"
-mi_theap_t*   _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld);
-void          _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock);
-void          _mi_theap_default_set(mi_theap_t* theap);
-void          _mi_theap_cached_set(mi_theap_t* theap);
-void          _mi_theap_collect_retired(mi_theap_t* theap, bool force);
-void          _mi_theap_collect_abandon(mi_theap_t* theap);
-bool          _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);
-void          _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page);
-bool          _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock);
-void          _mi_theap_incref(mi_theap_t* theap);
-void          _mi_theap_decref(mi_theap_t* theap);
-
-// "heap.c"
-void          _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);
-mi_decl_cold  mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap);  // get (and possible create) the theap belonging to a heap
-mi_decl_cold  mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap);     // get the theap for a heap without initializing (and return NULL in that case)
-void          _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to);  // in "arena.c"
-void          _mi_heap_destroy_pages(mi_heap_t* heap_from);                   // in "arena.c"
-void          _mi_heap_force_destroy(mi_heap_t* heap);                        // allow destroying the main heap
-
-// "stats.c"
-void          _mi_stats_init(void);
-void          _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from);
-
-mi_msecs_t    _mi_clock_now(void);
-mi_msecs_t    _mi_clock_end(mi_msecs_t start);
-mi_msecs_t    _mi_clock_start(void);
-
-// "alloc.c"
-void*         _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept;                  // called from `_mi_theap_malloc_aligned`
-void*         _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept;
-void*         _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept;     // called from `_mi_theap_malloc_aligned`
-void*         _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept;
-mi_block_t*   _mi_page_ptr_unalign(const mi_page_t* page, const void* p);
-void          _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size);
-
-#if MI_DEBUG>1
-bool          _mi_page_is_valid(mi_page_t* page);
-#endif
-
-
-// ------------------------------------------------------
-// Assertions
-// ------------------------------------------------------
-
-#if (MI_DEBUG)
-// use our own assertion to print without memory allocation
-mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept;
-#define mi_assert(expr)     ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__))
-#else
-#define mi_assert(x)
-#endif
-
-#if (MI_DEBUG>1)
-#define mi_assert_internal    mi_assert
-#else
-#define mi_assert_internal(x)
-#endif
-
-#if (MI_DEBUG>2)
-#define mi_assert_expensive   mi_assert
-#else
-#define mi_assert_expensive(x)
-#endif
-
-
-/* -----------------------------------------------------------
-  Statistics (in `stats.c`)
------------------------------------------------------------ */
-
-// add to stat keeping track of the peak
-void __mi_stat_increase(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount);
-
-// adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total)
-void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount);
-
-// counters can just be increased
-void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
-void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);
-
-#define mi_heap_stat_counter_increase(heap,stat,amount)         __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount)
-#define mi_heap_stat_increase(heap,stat,amount)                 __mi_stat_increase_mt( &(heap)->stats.stat, amount)
-#define mi_heap_stat_decrease(heap,stat,amount)                 __mi_stat_decrease_mt( &(heap)->stats.stat, amount)
-#define mi_heap_stat_adjust_increase(heap,stat,amnt)            __mi_stat_adjust_increase_mt( &(heap)->stats.stat, amnt)
-#define mi_heap_stat_adjust_decrease(heap,stat,amnt)            __mi_stat_adjust_decrease_mt( &(heap)->stats.stat, amnt)
-
-#define mi_subproc_stat_counter_increase(subproc,stat,amount)   __mi_stat_counter_increase_mt( &(subproc)->stats.stat, amount)
-#define mi_subproc_stat_increase(subproc,stat,amount)           __mi_stat_increase_mt( &(subproc)->stats.stat, amount)
-#define mi_subproc_stat_decrease(subproc,stat,amount)           __mi_stat_decrease_mt( &(subproc)->stats.stat, amount)
-#define mi_subproc_stat_adjust_increase(subproc,stat,amount)    __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amount)
-#define mi_subproc_stat_adjust_decrease(subproc,stat,amount)    __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount)
-
-#define mi_os_stat_counter_increase(stat,amount)                mi_subproc_stat_counter_increase(_mi_subproc(),stat,amount)
-#define mi_os_stat_increase(stat,amount)                        mi_subproc_stat_increase(_mi_subproc(),stat,amount)
-#define mi_os_stat_decrease(stat,amount)                        mi_subproc_stat_decrease(_mi_subproc(),stat,amount)
-
-#define mi_theap_stat_counter_increase(theap,stat,amount)       __mi_stat_counter_increase( &(theap)->stats.stat, amount)
-#define mi_theap_stat_increase(theap,stat,amount)               __mi_stat_increase( &(theap)->stats.stat, amount)
-#define mi_theap_stat_decrease(theap,stat,amount)               __mi_stat_decrease( &(theap)->stats.stat, amount)
-#define mi_theap_stat_adjust_increase(theap,stat,amnt)          __mi_stat_adjust_increase( &(theap)->stats.stat, amnt)
-#define mi_theap_stat_adjust_decrease(theap,stat,amnt)          __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt)
-
-
-/* -----------------------------------------------------------
-  Options (exposed for the debugger)
------------------------------------------------------------ */
-typedef enum mi_option_init_e {
-  MI_OPTION_UNINIT,       // not yet initialized
-  MI_OPTION_DEFAULTED,    // not found in the environment, use default value
-  MI_OPTION_INITIALIZED   // found in environment or set explicitly
-} mi_option_init_t;
-
-typedef struct mi_option_desc_s {
-  long              value;  // the value
-  mi_option_init_t  init;   // is it initialized yet? (from the environment)
-  mi_option_t       option; // for debugging: the option index should match the option
-  const char*       name;   // option name without `mimalloc_` prefix
-  const char*       legacy_name; // potential legacy option name
-} mi_option_desc_t;
-
-
-
-/* -----------------------------------------------------------
-  Inlined definitions
------------------------------------------------------------ */
-#define MI_UNUSED(x)     (void)(x)
-#if (MI_DEBUG>1)
-#define MI_UNUSED_RELEASE(x)
-#else
-#define MI_UNUSED_RELEASE(x)  MI_UNUSED(x)
-#endif
-
-#define MI_INIT4(x)   x(),x(),x(),x()
-#define MI_INIT8(x)   MI_INIT4(x),MI_INIT4(x)
-#define MI_INIT16(x)  MI_INIT8(x),MI_INIT8(x)
-#define MI_INIT32(x)  MI_INIT16(x),MI_INIT16(x)
-#define MI_INIT64(x)  MI_INIT32(x),MI_INIT32(x)
-#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x)
-#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x)
-
-#define MI_INIT74(x)  MI_INIT64(x),MI_INIT8(x),x(),x()
-#define MI_INIT5(x)   MI_INIT4(x),x()
-#define MI_INIT6(x)   MI_INIT4(x),x(),x()
-
-#include <string.h>
-// initialize a local variable to zero; use memset as compilers optimize constant sized memset's
-#define _mi_memzero_var(x)  memset(&x,0,sizeof(x))
-
-// Is `x` a power of two? (0 is considered a power of two)
-static inline bool _mi_is_power_of_two(uintptr_t x) {
-  return ((x & (x - 1)) == 0);
-}
-
-// Is a pointer aligned?
-static inline bool _mi_is_aligned(const void* p, size_t alignment) {
-  return (alignment==0 || ((uintptr_t)p % alignment) == 0);
-}
-
-// Align upwards
-static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
-  mi_assert_internal(alignment != 0);
-  uintptr_t mask = alignment - 1;
-  if ((alignment & mask) == 0) {  // power of two?
-    return ((sz + mask) & ~mask);
-  }
-  else {
-    return (((sz + mask)/alignment)*alignment);
-  }
-}
-
-// Align a pointer upwards
-static inline void* _mi_align_up_ptr(const void* p, size_t alignment) {
-  return (void*)_mi_align_up((uintptr_t)p, alignment);
-}
-
-static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
-  mi_assert_internal(alignment != 0);
-  uintptr_t mask = alignment - 1;
-  if ((alignment & mask) == 0) { // power of two?
-    return (sz & ~mask);
-  }
-  else {
-    return ((sz / alignment) * alignment);
-  }
-}
-
-// align a pointer downwards
-static inline void* _mi_align_down_ptr(const void* p, size_t alignment) {
-  return (void*)_mi_align_down((uintptr_t)p, alignment);
-}
-
-// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`.
-static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) {
-  mi_assert_internal(divider != 0);
-  return (divider == 0 ? size : ((size + divider - 1) / divider));
-}
-
-
-// clamp an integer
-static inline size_t _mi_clamp(size_t sz, size_t min, size_t max) {
-  if (sz < min) return min;
-  else if (sz > max) return max;
-  else return sz;
-}
-
-// Is memory zero initialized?
-static inline bool mi_mem_is_zero(const void* p, size_t size) {
-  for (size_t i = 0; i < size; i++) {
-    if (((uint8_t*)p)[i] != 0) return false;
-  }
-  return true;
-}
-
-// Align a byte size to a size in _machine words_,
-// i.e. byte size == `wsize*sizeof(void*)`.
-static inline size_t _mi_wsize_from_size(size_t size) {
-  mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t));
-  return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);
-}
-
-// Overflow detecting multiply
-#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5))
-#include <limits.h>      // UINT_MAX, ULONG_MAX
-#if defined(_CLOCK_T)    // for Illumos
-#undef _CLOCK_T
-#endif
-static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
-  #if (SIZE_MAX == ULONG_MAX)
-    return __builtin_umull_overflow(count, size, (unsigned long *)total);
-  #elif (SIZE_MAX == UINT_MAX)
-    return __builtin_umul_overflow(count, size, (unsigned int *)total);
-  #else
-    return __builtin_umulll_overflow(count, size, (unsigned long long *)total);
-  #endif
-}
-#else /* __builtin_umul_overflow is unavailable */
-static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
-  *total = count*size;
-  if mi_likely(((size|count)>>(4*MI_SIZE_SIZE))==0) {  // did size and count fit both in the lower half bits of a size_t?
-    return false;
-  }
-  else {
-    return (size!=0 && (SIZE_MAX / size) < count);
-  }
-}
-#endif
-
-// Safe multiply `count*size` into `total`; return `true` on overflow.
-static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) {
-  if (count==1) {  // quick check for the case where count is one (common for C++ allocators)
-    *total = size;
-    return false;
-  }
-  else if mi_likely(!mi_mul_overflow(count, size, total)) {
-    return false;
-  }
-  else {
-    #if MI_DEBUG > 0
-    _mi_error_message(EOVERFLOW, "allocation request is too large (%zu * %zu bytes)\n", count, size);
-    #endif
-    *total = SIZE_MAX;
-    return true;
-  }
-}
-
-
-/*----------------------------------------------------------------------------------------
-  Heap functions
-------------------------------------------------------------------------------------------- */
-
-extern mi_decl_hidden const mi_theap_t _mi_theap_empty;       // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_THREAD_LOCAL)
-extern mi_decl_hidden const mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated
-
-
-static inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) {
-  return mi_atomic_load_ptr_acquire(mi_heap_t,&theap->heap);
-}
-
-static inline bool mi_theap_is_initialized(const mi_theap_t* theap) {
-  return (theap != NULL && _mi_theap_heap(theap) != NULL);
-}
-
-static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) {
-  mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));
-  const size_t idx = _mi_wsize_from_size(size);
-  mi_assert_internal(idx < MI_PAGES_DIRECT);
-  return theap->pages_free_direct[idx];
-}
-
-
-//static inline uintptr_t _mi_ptr_cookie(const void* p) {
-//  extern mi_theap_t _mi_theap_main;
-//  mi_assert_internal(_mi_theap_main.cookie != 0);
-//  return ((uintptr_t)p ^ _mi_theap_main.cookie);
-//}
-
-
-/* -----------------------------------------------------------
-  The page map maps addresses to `mi_page_t` pointers
------------------------------------------------------------ */
-
-#if MI_PAGE_MAP_FLAT
-
-// flat page-map committed on demand, using one byte per slice (64 KiB).
-// single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses)
-// used by default on <= 40 bit virtual address spaces.
-extern mi_decl_hidden uint8_t* _mi_page_map;
-
-static inline size_t _mi_page_map_index(const void* p) {
-  return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT);
-}
-
-static inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) {
-  const size_t idx = _mi_page_map_index(p);
-  const size_t ofs = _mi_page_map[idx];
-  if (valid != NULL) { *valid = (ofs != 0); }
-  return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT);
-}
-
-static inline mi_page_t* _mi_checked_ptr_page(const void* p) {
-  bool valid;
-  mi_page_t* const page = _mi_ptr_page_ex(p, &valid);
-  return (valid ? page : NULL);
-}
-
-static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {
-  return _mi_ptr_page_ex(p, NULL);
-}
-
-#else
-
-// 2-level page map:
-// double indirection, but low commit and low virtual reserve.
-//
-// the page-map is usually 4 MiB (for 48 bit virtual addresses) and points to sub maps of 64 KiB.
-// the page-map is committed on-demand (in 64 KiB parts) (and sub-maps are committed on-demand as well)
-// one sub page-map = 64 KiB => covers 2^(16-3) * 2^16 = 2^29 = 512 MiB address space
-// the page-map needs 48-(16+13) = 19 bits => 2^19 sub map pointers = 2^22 bytes = 4 MiB reserved size.
-#define MI_PAGE_MAP_SUB_SHIFT     (13)
-#define MI_PAGE_MAP_SUB_COUNT     (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT)
-#define MI_PAGE_MAP_SHIFT         (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)
-#define MI_PAGE_MAP_COUNT         (MI_ZU(1) << MI_PAGE_MAP_SHIFT)
-
-typedef mi_page_t**   mi_submap_t;
-extern mi_decl_hidden _Atomic(mi_submap_t)* _mi_page_map;
-
-static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) {
-  const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE);
-  if (sub_idx != NULL) { *sub_idx = u % MI_PAGE_MAP_SUB_COUNT; }
-  return (u / MI_PAGE_MAP_SUB_COUNT);
-}
-
-static inline mi_submap_t _mi_page_map_at(size_t idx) {
-  return mi_atomic_load_ptr_relaxed(mi_page_t*, &_mi_page_map[idx]);
-}
-
-static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {
-  size_t sub_idx;
-  const size_t idx = _mi_page_map_index(p, &sub_idx);
-  return (_mi_page_map_at(idx))[sub_idx];  // NULL if p==NULL
-}
-
-static inline mi_page_t* _mi_checked_ptr_page(const void* p) {
-  size_t sub_idx;
-  const size_t idx = _mi_page_map_index(p, &sub_idx);
-  mi_submap_t const sub = _mi_page_map_at(idx);
-  if mi_unlikely(sub == NULL) return NULL;
-  return sub[sub_idx];
-}
-
-#endif
-
-
-static inline mi_page_t* _mi_ptr_page(const void* p) {
-  mi_assert_internal(p==NULL || mi_is_in_heap_region(p));
-  #if MI_DEBUG || MI_SECURE || defined(__APPLE__)
-  return _mi_checked_ptr_page(p);
-  #else
-  return _mi_unchecked_ptr_page(p);
-  #endif
-}
-
-
-// Get the block size of a page
-static inline size_t mi_page_block_size(const mi_page_t* page) {
-  mi_assert_internal(page->block_size > 0);
-  return page->block_size;
-}
-
-// Page start
-static inline uint8_t* mi_page_start(const mi_page_t* page) {
-  return page->page_start;
-}
-
-static inline size_t mi_page_size(const mi_page_t* page) {
-  return mi_page_block_size(page) * page->reserved;
-}
-
-static inline uint8_t* mi_page_area(const mi_page_t* page, size_t* size) {
-  if (size) { *size = mi_page_size(page); }
-  return mi_page_start(page);
-}
-
-static inline size_t mi_page_info_size(void) {
-  return _mi_align_up(sizeof(mi_page_t), MI_MAX_ALIGN_SIZE);
-}
-
-static inline bool mi_page_contains_address(const mi_page_t* page, const void* p) {
-  size_t psize;
-  uint8_t* start = mi_page_area(page, &psize);
-  return (start <= (uint8_t*)p && (uint8_t*)p < start + psize);
-}
-
-static inline bool mi_page_is_in_arena(const mi_page_t* page) {
-  return (page->memid.memkind == MI_MEM_ARENA);
-}
-
-static inline bool mi_page_is_singleton(const mi_page_t* page) {
-  return (page->reserved == 1);
-}
-
-// Get the usable block size of a page without fixed padding.
-// This may still include internal padding due to alignment and rounding up size classes.
-static inline size_t mi_page_usable_block_size(const mi_page_t* page) {
-  return mi_page_block_size(page) - MI_PADDING_SIZE;
-}
-
-static inline bool mi_page_meta_is_separated(const mi_page_t* page) {
-  #if MI_PAGE_META_IS_SEPARATED
-  // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL
-  return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN));
-  #else
-  MI_UNUSED(page);
-  return false;  
-  #endif
-}
-
-static inline uint8_t* mi_page_slice_start(const mi_page_t* page) {
-  if (mi_page_meta_is_separated(page)) {  
-    // page meta info is at a separate location (at `arena->pages`)
-    return (uint8_t*)_mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN);
-  }
-  else {
-    // page meta info is at the start of the page slices
-    return (uint8_t*)page;
-  }
-}
-
-// This gives the offset relative to the start slice of a page. 
-static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) {
-  return (page->page_start - mi_page_slice_start(page)) + offset_relative_to_page_start;
-}
-
-// Currently committed part of a page
-static inline size_t mi_page_committed(const mi_page_t* page) {
-  return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - mi_page_slice_offset_of(page,0));
-}
-
-// are all blocks in a page freed?
-// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`.
-static inline bool mi_page_all_free(const mi_page_t* page) {
-  mi_assert_internal(page != NULL);
-  return (page->used == 0);
-}
-
-// are there immediately available blocks, i.e. blocks available on the free list.
-static inline bool mi_page_immediate_available(const mi_page_t* page) {
-  mi_assert_internal(page != NULL);
-  return (page->free != NULL);
-}
-
-
-// is the page not yet used up to its reserved space?
-static inline bool mi_page_is_expandable(const mi_page_t* page) {
-  mi_assert_internal(page != NULL);
-  mi_assert_internal(page->capacity <= page->reserved);
-  return (page->capacity < page->reserved);
-}
-
-
-static inline bool mi_page_is_full(const mi_page_t* page) {
-  const bool full = (page->reserved == page->used);
-  mi_assert_internal(!full || page->free == NULL);
-  return full;
-}
-
-// is more than 7/8th of a page in use?
-static inline bool mi_page_is_mostly_used(const mi_page_t* page) {
-  if (page==NULL) return true;
-  uint16_t frac = page->reserved / 8U;
-  return (page->reserved - page->used <= frac);
-}
-
-// is more than (n-1)/n'th of a page in use?
-static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) {
-  if (page==NULL) return true;
-  uint16_t frac = page->reserved / n;
-  return (page->reserved - page->used <= frac);
-}
-
-
-static inline bool mi_page_is_huge(const mi_page_t* page) {
-  return (mi_page_is_singleton(page) &&
-          (page->block_size > MI_LARGE_MAX_OBJ_SIZE ||
-           (mi_memkind_is_os(page->memid.memkind) && page->memid.mem.os.base < (void*)page)));
-}
-
-static inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t size) {
-  mi_page_queue_t* const pq = &((mi_theap_t*)theap)->pages[_mi_bin(size)];
-  if (size <= MI_LARGE_MAX_OBJ_SIZE) { mi_assert_internal(pq->block_size <= MI_LARGE_MAX_OBJ_SIZE); }
-  return pq;
-}
-
-
-//-----------------------------------------------------------
-// Page thread id and flags
-//-----------------------------------------------------------
-
-// Thread id of thread that owns this page (with flags in the bottom 2 bits)
-static inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) {
-  return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id);
-}
-
-// Plain thread id of the thread that owns this page
-static inline mi_threadid_t mi_page_thread_id(const mi_page_t* page) {
-  return (mi_page_xthread_id(page) & ~MI_PAGE_FLAG_MASK);
-}
-
-static inline mi_page_flags_t mi_page_flags(const mi_page_t* page) {
-  return (mi_page_xthread_id(page) & MI_PAGE_FLAG_MASK);
-}
-
-static inline bool mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) {
-  mi_page_flags_t old;
-  if (set) { old = mi_atomic_or_relaxed(&page->xthread_id, newflag); }
-      else { old = mi_atomic_and_relaxed(&page->xthread_id, ~newflag); }
-  return ((old & newflag) == newflag);
-}
-
-static inline bool mi_page_is_in_full(const mi_page_t* page) {
-  return ((mi_page_flags(page) & MI_PAGE_IN_FULL_QUEUE) != 0);
-}
-
-static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {
-  const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE);
-  if (was_in_full != in_full) {
-    // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220)
-    mi_theap_t* const theap = page->theap;
-    mi_assert_internal(theap!=NULL);
-    if (theap != NULL) {
-      const size_t size = page->capacity * mi_page_block_size(page);
-      if (in_full) { theap->pages_full_size += size; }
-              else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; }
-    }
-  }
-}
-
-static inline bool mi_page_has_interior_pointers(const mi_page_t* page) {
-  return ((mi_page_flags(page) & MI_PAGE_HAS_INTERIOR_POINTERS) != 0);
-}
-
-static inline void mi_page_set_has_interior_pointers(mi_page_t* page, bool has_aligned) {
-  mi_page_flags_set(page, has_aligned, MI_PAGE_HAS_INTERIOR_POINTERS);
-}
-
-static inline void mi_page_set_theap(mi_page_t* page, mi_theap_t* theap) {
-  // mi_assert_internal(!mi_page_is_in_full(page));  // can happen when destroying pages on theap_destroy
-  page->theap = theap;
-  const mi_threadid_t tid = (theap == NULL ? MI_THREADID_ABANDONED : theap->tld->thread_id);
-  mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0);
-
-  // we need to use an atomic cas since a concurrent thread may still set the MI_PAGE_HAS_INTERIOR_POINTERS flag (see `alloc_aligned.c`).
-  mi_threadid_t xtid_old = mi_page_xthread_id(page);
-  mi_threadid_t xtid;
-  do {
-    xtid = tid | (xtid_old & MI_PAGE_FLAG_MASK);
-  } while (!mi_atomic_cas_weak_release(&page->xthread_id, &xtid_old, xtid));
-}
-
-static inline bool mi_page_is_abandoned(const mi_page_t* page) {
-  // note: the xtheap field of an abandoned theap is set to the subproc (for fast reclaim-on-free)
-  return (mi_page_thread_id(page) <= MI_THREADID_ABANDONED_MAPPED);
-}
-
-static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) {
-  return (mi_page_thread_id(page) == MI_THREADID_ABANDONED_MAPPED);
-}
-
-static inline void mi_page_set_abandoned_mapped(mi_page_t* page) {
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_atomic_or_relaxed(&page->xthread_id, (mi_threadid_t)MI_THREADID_ABANDONED_MAPPED);
-}
-
-static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) {
-  mi_assert_internal(mi_page_is_abandoned_mapped(page));
-  mi_atomic_and_relaxed(&page->xthread_id, (mi_threadid_t)MI_PAGE_FLAG_MASK);
-}
-
-
-static inline mi_theap_t* mi_page_theap(const mi_page_t* page) {
-  mi_assert_internal(!mi_page_is_abandoned(page));
-  mi_assert_internal(page->theap != NULL);
-  return page->theap;
-}
-
-static inline mi_tld_t* mi_page_tld(const mi_page_t* page) {
-  mi_assert_internal(!mi_page_is_abandoned(page));
-  mi_assert_internal(page->theap != NULL);
-  return page->theap->tld;
-}
-
-
-static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {
-  mi_heap_t* heap = page->heap;
-  // we use NULL for the main heap to make `_mi_page_get_associated_theap` fast in `free.c:mi_abandoned_page_try_reclaim`.
-  if mi_likely(heap==NULL) heap = mi_heap_main();
-  mi_assert_internal(heap != NULL);
-  return heap;
-}
-
-//-----------------------------------------------------------
-// Thread free list and ownership
-//-----------------------------------------------------------
-
-// Thread free flag helpers
-static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) {
-  return (mi_block_t*)(tf & ~1);
-}
-static inline bool mi_tf_is_owned(mi_thread_free_t tf) {
-  return ((tf & 1) == 1);
-}
-static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) {
-  return (mi_thread_free_t)((uintptr_t)block | (owned ? 1 : 0));
-}
-
-// Thread free access
-static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {
-  return mi_tf_block(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free));
-}
-
-// are there any available blocks?
-static inline bool mi_page_has_any_available(const mi_page_t* page) {
-  mi_assert_internal(page != NULL && page->reserved > 0);
-  return (page->used < page->reserved || (mi_page_thread_free(page) != NULL));
-}
-
-// Owned?
-static inline bool mi_page_is_owned(const mi_page_t* page) {
-  return mi_tf_is_owned(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free));
-}
-
-// get ownership; returns true if the page was not owned before.
-static inline bool mi_page_claim_ownership(mi_page_t* page) {
-  const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, (uintptr_t)1);
-  return ((old&1)==0);
-}
-
-
-/* -------------------------------------------------------------------
-  Guarded objects
-------------------------------------------------------------------- */
-#if MI_GUARDED
-// we always align guarded pointers in a block at an offset
-// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones
-#define MI_BLOCK_TAG_ALIGNED   ((mi_encoded_t)(0))
-#define MI_BLOCK_TAG_GUARDED   (~MI_BLOCK_TAG_ALIGNED)
-#endif
-
-static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) {
-#if MI_GUARDED
-  const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block;
-  return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED);
-#else
-  MI_UNUSED(block); MI_UNUSED(p);
-  return false;
-#endif  
-}
-
-#if MI_GUARDED
-static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) {
-  // this code is written to result in fast assembly as it is on the hot path for allocation
-  const size_t count = theap->guarded_sample_count - 1;  // if the rate was 0, this will underflow and count for a long time..
-  if mi_likely(count != 0) {
-    // no sample
-    theap->guarded_sample_count = count;
-    return false;
-  }
-  else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) {
-    // use guarded allocation
-    theap->guarded_sample_count = theap->guarded_sample_rate;  // reset
-    return (theap->guarded_sample_rate != 0);
-  }
-  else {
-    // failed size criteria, rewind count (but don't write to an empty theap)
-    if (theap->guarded_sample_rate != 0) { theap->guarded_sample_count = 1; }
-    return false;
-  }
-}
-
-mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;
-
-#endif
-
-
-/* -------------------------------------------------------------------
-Encoding/Decoding the free list next pointers
-
-This is to protect against buffer overflow exploits where the
-free list is mutated. Many hardened allocators xor the next pointer `p`
-with a secret key `k1`, as `p^k1`. This prevents overwriting with known
-values but might be still too weak: if the attacker can guess
-the pointer `p` this  can reveal `k1` (since `p^k1^p == k1`).
-Moreover, if multiple blocks can be read as well, the attacker can
-xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot
-about the pointers (and subsequently `k1`).
-
-Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<<k1)+k1`.
-Since these operations are not associative, the above approaches do not
-work so well any more even if the `p` can be guesstimated. For example,
-for the read case we can subtract two entries to discard the `+k1` term,
-but that leads to `((p1^k2)<<<k1) - ((p2^k2)<<<k1)` at best.
-We include the left-rotation since xor and addition are otherwise linear
-in the lowest bit. Finally, both keys are unique per page which reduces
-the re-use of keys by a large factor.
-
-We also pass a separate `null` value to be used as `NULL` or otherwise
-`(k2<<<k1)+k1` would appear (too) often as a sentinel value.
-------------------------------------------------------------------- */
-
-static inline bool mi_is_in_same_page(const void* p, const void* q) {
-  mi_page_t* page = _mi_ptr_page(p);
-  return mi_page_contains_address(page,q);
-  // return (_mi_ptr_page(p) == _mi_ptr_page(q));
-}
-
-static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) {
-  void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]);
-  return (p==null ? NULL : p);
-}
-
-static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) {
-  uintptr_t x = (uintptr_t)(p==NULL ? null : p);
-  return mi_rotl(x ^ keys[1], keys[0]) + keys[0];
-}
-
-static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) {
-  const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys));
-  // make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951)
-  #if MI_BIG_ENDIAN
-  return (x & 0x00FFFFFF);
-  #else
-  return (x & 0xFFFFFF00);
-  #endif
-}
-
-static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) {
-  mi_track_mem_defined(block,sizeof(mi_block_t));
-  mi_block_t* next;
-  #if MI_ENCODE_FREELIST
-  next = (mi_block_t*)mi_ptr_decode(null, block->next, keys);
-  #else
-  MI_UNUSED(keys); MI_UNUSED(null);
-  next = (mi_block_t*)block->next;
-  #endif
-  mi_track_mem_noaccess(block,sizeof(mi_block_t));
-  return next;
-}
-
-static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) {
-  mi_track_mem_undefined(block,sizeof(mi_block_t));
-  #if MI_ENCODE_FREELIST
-  block->next = mi_ptr_encode(null, next, keys);
-  #else
-  MI_UNUSED(keys); MI_UNUSED(null);
-  block->next = (mi_encoded_t)next;
-  #endif
-  mi_track_mem_noaccess(block,sizeof(mi_block_t));
-}
-
-static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) {
-  #if MI_ENCODE_FREELIST
-  mi_block_t* next = mi_block_nextx(page,block,page->keys);
-  // check for free list corruption: is `next` at least in the same page?
-  // TODO: check if `next` is `page->block_size` aligned?
-  if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) {
-    _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next);
-    next = NULL;
-  }
-  return next;
-  #else
-  MI_UNUSED(page);
-  return mi_block_nextx(page,block,NULL);
-  #endif
-}
-
-static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) {
-  #if MI_ENCODE_FREELIST
-  mi_block_set_nextx(page,block,next, page->keys);
-  #else
-  MI_UNUSED(page);
-  mi_block_set_nextx(page,block,next,NULL);
-  #endif
-}
-
-/* -----------------------------------------------------------
-  arena blocks
------------------------------------------------------------ */
-
-// Blocks needed for a given byte size
-static inline size_t mi_slice_count_of_size(size_t size) {
-  return _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
-}
-
-// Byte size of a number of blocks
-static inline size_t mi_size_of_slices(size_t bcount) {
-  return (bcount * MI_ARENA_SLICE_SIZE);
-}
-
-
-/* -----------------------------------------------------------
-  memory id's
------------------------------------------------------------ */
-
-static inline mi_memid_t _mi_memid_create(mi_memkind_t memkind) {
-  mi_memid_t memid;
-  _mi_memzero_var(memid);
-  memid.memkind = memkind;
-  return memid;
-}
-
-static inline mi_memid_t _mi_memid_none(void) {
-  return _mi_memid_create(MI_MEM_NONE);
-}
-
-static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool committed, bool is_zero, bool is_large) {
-  mi_memid_t memid = _mi_memid_create(MI_MEM_OS);
-  memid.mem.os.base = base;
-  memid.mem.os.size = size;
-  memid.initially_committed = committed;
-  memid.initially_zero = is_zero;
-  memid.is_pinned = is_large;
-  return memid;
-}
-
-static inline mi_memid_t _mi_memid_create_meta(void* mpage, size_t block_idx, size_t block_count) {
-  mi_memid_t memid = _mi_memid_create(MI_MEM_META);
-  memid.mem.meta.meta_page = mpage;
-  memid.mem.meta.block_index = (uint32_t)block_idx;
-  memid.mem.meta.block_count = (uint32_t)block_count;
-  memid.initially_committed = true;
-  memid.initially_zero = true;
-  memid.is_pinned = true;
-  return memid;
-}
-
-
-// -------------------------------------------------------------------
-// Fast "random" shuffle
-// -------------------------------------------------------------------
-
-static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
-  if (x==0) { x = 17; }   // ensure we don't get stuck in generating zeros
-#if (MI_INTPTR_SIZE>=8)
-  // by Sebastiano Vigna, see: <http://xoshiro.di.unimi.it/splitmix64.c>
-  x ^= x >> 30;
-  x *= 0xbf58476d1ce4e5b9UL;
-  x ^= x >> 27;
-  x *= 0x94d049bb133111ebUL;
-  x ^= x >> 31;
-#elif (MI_INTPTR_SIZE==4)
-  // by Chris Wellons, see: <https://nullprogram.com/blog/2018/07/31/>
-  x ^= x >> 16;
-  x *= 0x7feb352dUL;
-  x ^= x >> 15;
-  x *= 0x846ca68bUL;
-  x ^= x >> 16;
-#endif
-  return x;
-}
-
-
-// ---------------------------------------------------------------------------------
-// Provide our own `_mi_memcpy/set` for potential performance optimizations.
-//
-// For now, only on x64/x86 we optimize to `rep movsb/stosb`.
-// Generally, we check for "fast short rep movsb/stosb" (FSRM/FSRS) or "fast enhanced rep movsb" (ERMS) support
-// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253.
-// Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows.
-// ---------------------------------------------------------------------------------
-
-#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__))
-
-extern mi_decl_hidden size_t _mi_cpu_movsb_max;  // in init.c
-extern mi_decl_hidden size_t _mi_cpu_stosb_max;
-
-static inline void mi_rep_movsb(void* dst, const void* src, size_t n) {
-  #if defined(__GNUC__)
-  __asm volatile("rep movsb" : "+D"(dst), "+c"(n), "+S"(src) : : "memory");
-  #else
-  __movsb((unsigned char*)dst, (const unsigned char*)src, n);
-  #endif
-}
-
-static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) {
-  #if defined(__GNUC__)
-  __asm volatile("rep stosb" : "+D"(dst), "+c"(n) : "a"(val) : "memory");
-  #else
-  __stosb((unsigned char*)dst, val, n);
-  #endif
-}
-
-static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
-  if mi_likely(n <= _mi_cpu_movsb_max) {  // has fsrm && n <= 127  (todo: and maybe has erms?)
-    mi_rep_movsb(dst, src, n);
-  }
-  else {
-    memcpy(dst, src, n);
-  }
-}
-
-static inline void _mi_memset(void* dst, int val, size_t n) {
-  if mi_likely(n <= _mi_cpu_stosb_max) {  // has fsrs && n <= 127
-    mi_rep_stosb(dst, (uint8_t)val, n);
-  }
-  else {
-    memset(dst, val, n);
-  }
-}
-
-#else
-
-static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
-  memcpy(dst, src, n);
-}
-
-static inline void _mi_memset(void* dst, int val, size_t n) {
-  memset(dst, val, n);
-}
-
-#endif
-
-// -------------------------------------------------------------------------------
-// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned
-// This is used for example in `mi_realloc`.
-// -------------------------------------------------------------------------------
-
-#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
-
-// On GCC/CLang we provide a hint that the pointers are word aligned.
-static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
-  mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
-  void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
-  const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);
-  _mi_memcpy(adst, asrc, n);
-}
-
-static inline void _mi_memset_aligned(void* dst, int val, size_t n) {
-  mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
-  void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
-  _mi_memset(adst, val, n);
-}
-
-#else
-
-// Default fallback on `_mi_memcpy`
-static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
-  mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
-  _mi_memcpy(dst, src, n);
-}
-
-static inline void _mi_memset_aligned(void* dst, int val, size_t n) {
-  mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
-  _mi_memset(dst, val, n);
-}
-
-#endif
-
-static inline void _mi_memzero(void* dst, size_t n) {
-  _mi_memset(dst, 0, n);
-}
-
-static inline void _mi_memzero_aligned(void* dst, size_t n) {
-  _mi_memset_aligned(dst, 0, n);
-}
-
-
-
-#endif  // MI_INTERNAL_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/prim.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/prim.h
deleted file mode 100644
index aecfc807..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/prim.h
+++ /dev/null
@@ -1,542 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MIMALLOC_PRIM_H
-#define MIMALLOC_PRIM_H
-#include "internal.h"             // mi_decl_hidden
-
-// --------------------------------------------------------------------------
-// This file specifies the primitive portability API.
-// Each OS/host needs to implement these primitives, see `src/prim`
-// for implementations on Window, macOS, WASI, and Linux/Unix.
-//
-// note: on all primitive functions, we always have result parameters != NULL, and:
-//  addr != NULL and page aligned
-//  size > 0     and page aligned
-//  the return value is an error code as an `int` where 0 is success
-// --------------------------------------------------------------------------
-
-// OS memory configuration
-typedef struct mi_os_mem_config_s {
-  size_t  page_size;              // default to 4KiB
-  size_t  large_page_size;        // 0 if not supported, usually 2MiB (4MiB on Windows)
-  size_t  alloc_granularity;      // smallest allocation size (usually 4KiB, on Windows 64KiB)
-  size_t  physical_memory_in_kib; // physical memory size in KiB
-  size_t  virtual_address_bits;   // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization)
-  bool    has_overcommit;         // can we reserve more memory than can be actually committed?
-  bool    has_partial_free;       // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc)
-  bool    has_virtual_reserve;    // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)
-  bool    has_transparent_huge_pages;  // true if transparent huge pages are enabled (on Linux)
-} mi_os_mem_config_t;
-
-// Initialize
-void _mi_prim_mem_init( mi_os_mem_config_t* config );
-
-// Free OS memory
-int _mi_prim_free(void* addr, size_t size );
-
-// Allocate OS memory. Return NULL on error.
-// The `try_alignment` is just a hint and the returned pointer does not have to be aligned.
-// If `commit` is false, the virtual memory range only needs to be reserved (with no access)
-// which will later be committed explicitly using `_mi_prim_commit`.
-// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
-// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored.
-// pre: !commit => !allow_large
-//      try_alignment >= _mi_os_page_size() and a power of 2
-int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr);
-
-// Commit memory. Returns error code or 0 on success.
-// For example, on Linux this would make the memory PROT_READ|PROT_WRITE.
-// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows)
-int _mi_prim_commit(void* addr, size_t size, bool* is_zero);
-
-// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true
-// if the memory would need to be re-committed. For example, on Windows this is always true,
-// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit.
-// pre: needs_recommit != NULL
-int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit);
-
-// Reset memory. The range keeps being accessible but the content might be reset to zero at any moment.
-// Returns error code or 0 on success.
-int _mi_prim_reset(void* addr, size_t size);
-
-// Reuse memory. This is called for memory that is already committed but
-// may have been reset (`_mi_prim_reset`) or decommitted (`_mi_prim_decommit`) where `needs_recommit` was false.
-// Returns error code or 0 on success. On most platforms this is a no-op.
-int _mi_prim_reuse(void* addr, size_t size);
-
-// Protect memory. Returns error code or 0 on success.
-int _mi_prim_protect(void* addr, size_t size, bool protect);
-
-// Allocate huge (1GiB) pages possibly associated with a NUMA node.
-// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
-// pre: size > 0  and a multiple of 1GiB.
-//      numa_node is either negative (don't care), or a numa node number.
-int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);
-
-// Return the current NUMA node
-size_t _mi_prim_numa_node(void);
-
-// Return the number of logical NUMA nodes
-size_t _mi_prim_numa_node_count(void);
-
-// Clock ticks
-mi_msecs_t _mi_prim_clock_now(void);
-
-// Return process information (only for statistics)
-typedef struct mi_process_info_s {
-  mi_msecs_t  elapsed;
-  mi_msecs_t  utime;
-  mi_msecs_t  stime;
-  size_t      current_rss;
-  size_t      peak_rss;
-  size_t      current_commit;
-  size_t      peak_commit;
-  size_t      page_faults;
-} mi_process_info_t;
-
-void _mi_prim_process_info(mi_process_info_t* pinfo);
-
-// Default stderr output. (only for warnings etc. with verbose enabled)
-// msg != NULL && _mi_strlen(msg) > 0
-void _mi_prim_out_stderr( const char* msg );
-
-// Get an environment variable. (only for options)
-// name != NULL, result != NULL, result_size >= 64
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size);
-
-
-// Fill a buffer with strong randomness; return `false` on error or if
-// there is no strong randomization available.
-bool _mi_prim_random_buf(void* buf, size_t buf_len);
-
-// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination.
-void _mi_prim_thread_init_auto_done(void);
-
-// Called on process exit and may take action to clean up resources associated with the thread auto done.
-void _mi_prim_thread_done_auto_done(void);
-
-// Called when the default theap for a thread changes
-void _mi_prim_thread_associate_default_theap(mi_theap_t* theap);
-
-// Is this thread part of a thread pool?
-bool _mi_prim_thread_is_in_threadpool(void);
-
-// Yield to other threads. Should be similar to `sleep(0)`. 
-// Is called only in rare situations and does not have to be lightning fast.
-void _mi_prim_thread_yield(void);
-
-//-------------------------------------------------------------------
-// Access to TLS (thread local storage) slots.
-// We need fast access to both a unique thread id (in `free.c:mi_free`) and
-// to a thread-local theap pointer (in `alloc.c:mi_malloc`).
-// To achieve this we use specialized code for various platforms.
-//-------------------------------------------------------------------
-
-// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.
-// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform.
-// If you test on another platform and it works please send a PR :-)
-// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.
-//
-// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly,
-// but unfortunately we can not detect support reliably (see issue #883)
-// We also use it on Apple OS as we use a TLS slot for the default theap there.
-#if (defined(_WIN32)) || \
-    (defined(__GNUC__) && ( \
-           (defined(__GLIBC__)   && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
-        || (defined(__APPLE__)   && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \
-        || (defined(__BIONIC__)  && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
-        || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
-        || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
-      ))
-
-static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {
-  void* res;
-  const size_t ofs = (slot*sizeof(void*));
-  #if defined(_WIN32)
-    #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)
-      res = (void*)__readgsqword((unsigned long)ofs);   // direct load at offset from gs
-    #elif _M_IX86 && !defined(_M_ARM64EC)
-      res = (void*)__readfsdword((unsigned long)ofs);   // direct load at offset from fs
-    #else
-      res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs);
-    #endif
-  #elif defined(__i386__)
-    __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : );  // x86 32-bit always uses GS
-  #elif defined(__APPLE__) && defined(__x86_64__)
-    __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : );  // x86_64 macOSX uses GS
-  #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
-    __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : );  // x32 ABI
-  #elif defined(__x86_64__)
-    __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : );  // x86_64 Linux, BSD uses FS
-  #elif defined(__arm__)
-    void** tcb; MI_UNUSED(ofs);
-    __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
-    res = tcb[slot];
-  #elif defined(__aarch64__)
-    void** tcb; MI_UNUSED(ofs);
-    #if defined(__APPLE__) // M1, issue #343
-    __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
-    #else
-    __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
-    #endif
-    res = tcb[slot];
-  #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
-    MI_UNUSED(ofs);
-    res = pthread_getspecific(slot);
-  #else
-    #define MI_HAS_TLS_SLOT 0
-    MI_UNUSED(ofs);
-    res = NULL;
-  #endif
-  return res;
-}
-
-#ifndef MI_HAS_TLS_SLOT
-#define MI_HAS_TLS_SLOT 1
-#endif
-
-// setting a tls slot is only used on macOS for now
-static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
-  const size_t ofs = (slot*sizeof(void*));
-  #if defined(_WIN32)
-    ((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs);
-  #elif defined(__i386__)
-    __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : );  // 32-bit always uses GS
-  #elif defined(__APPLE__) && defined(__x86_64__)
-    __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : );  // x86_64 macOS uses GS
-  #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
-    __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : );  // x32 ABI
-  #elif defined(__x86_64__)
-    __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : );  // x86_64 Linux, BSD uses FS
-  #elif defined(__arm__)
-    void** tcb; MI_UNUSED(ofs);
-    __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
-    tcb[slot] = value;
-  #elif defined(__aarch64__)
-    void** tcb; MI_UNUSED(ofs);
-    #if defined(__APPLE__) // M1, issue #343
-    __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
-    #else
-    __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
-    #endif
-    tcb[slot] = value;
-  #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
-    MI_UNUSED(ofs);
-    pthread_setspecific(slot, value);
-  #else
-    MI_UNUSED(ofs); MI_UNUSED(value);
-  #endif
-}
-
-#endif
-
-
-// defined in `init.c`; do not use these directly
-extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main;     // theap belonging to the main heap
-extern mi_decl_hidden bool _mi_process_is_initialized;                // has mi_process_init been called?
-
-
-//-------------------------------------------------------------------
-// Get a fast unique thread id.
-//
-// Getting the thread id should be performant as it is called in the
-// fast path of `_mi_free` and we specialize for various platforms as
-// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.
-// We only require _mi_prim_thread_id() to return a unique id
-// for each thread (unequal to zero).
-//-------------------------------------------------------------------
-
-
-// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id
-// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
-// Nevertheless, it seems needed on older graviton platforms (see issue #851).
-// For now, we only enable this for specific platforms.
-#if !defined(MI_USE_BUILTIN_THREAD_POINTER)   /* allow user override */
-  #if !defined(__APPLE__)  /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
-      && !defined(__CYGWIN__) \
-      && !defined(MI_LIBC_MUSL) \
-      && (!defined(__clang_major__) || __clang_major__ >= 14)  /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
-    #if    (defined(__GNUC__) && (__GNUC__ >= 7)  && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
-        || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
-        || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
-      #define MI_USE_BUILTIN_THREAD_POINTER  1
-    #endif
-  #endif
-#endif
-
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept;
-
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
-  const mi_threadid_t tid = __mi_prim_thread_id();
-  mi_assert_internal(tid > 1);
-  mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0);  // bottom 2 bits are clear?
-  return tid;
-}
-
-// Get a unique id for the current thread.
-#if defined(MI_PRIM_THREAD_ID)
-
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
-  const mi_threadid_t tid = MI_PRIM_THREAD_ID();  // used for example by CPython for a free threaded build (see python/cpython#115488)
-  mi_assert_internal( (tid & 0x03) == 0 );        // mimalloc reserves the bottom 2 bits
-  return tid;
-}
-
-#elif defined(_WIN32)
-
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
-  // Windows: works on Intel and ARM in both 32- and 64-bit
-  return (uintptr_t)NtCurrentTeb();
-}
-
-#elif MI_USE_BUILTIN_THREAD_POINTER
-
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
-  // Works on most Unix based platforms with recent compilers
-  return (uintptr_t)__builtin_thread_pointer();
-}
-
-#elif MI_HAS_TLS_SLOT
-
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
-  #if defined(__BIONIC__)
-    // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
-    // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
-    return (uintptr_t)mi_prim_tls_slot(1);
-  #else
-    // in all our other targets, slot 0 is the thread id
-    // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h
-    // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36
-    return (uintptr_t)mi_prim_tls_slot(0);
-  #endif
-}
-
-#else
-
-// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
-  return (uintptr_t)&__mi_theap_main;
-}
-
-#endif
-
-
-
-/* ----------------------------------------------------------------------------------------
-Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`).
-
-This is inlined here as it is on the fast path for allocation functions.
-We have 4 models:
-
-- MI_TLS_MODEL_THREAD_LOCAL: use regular thread local (default on Linux, FreeBSD, etc)
-    On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a
-    thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures
-    that the storage will always be available and properly initialized (with an empty theap).
-
-    On some platforms the underlying TLS implementation (or the loader) will call itself `malloc`
-    on a first access to a thread local and recurse in the MI_TLS_MODEL_THREAD_LOCAL.
-    A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process
-    is initialized before accessing the thread-local. This is a check in the fast path though
-    so this should be avoided.
-
-- MI_TLS_MODEL_FIXED_SLOT: use a fixed slot in the TLS block (default on macOS)
-    This reserves an unused and fixed TLS slot. This is fast and avoids the problem
-    where the underlying TLS implementation (or the loader) will call itself `malloc`
-    on a first access to a thread local (and recurse in the MI_TLS_MODEL_THREAD_LOCAL).
-    This goes wrong though if the OS or a library uses the same fixed slot.
-
-- MI_TLS_MODEL_DYNAMIC_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows)
-    Windows has somewhat slow thread locals so by default we use TlsAlloc'd slots which
-    can be more efficient. First tries to use one of the "direct" first 64 slots which 
-    are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots).
-    (If the allocated slot happens to always be under 64 for a particular program,
-    one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.)
-
-- MI_TLS_MODEL_DYNAMIC_PTHREADS: use `pthread_getspecific`. (default on OpenBSD, maybe good for Android as well?)
-    Use pthread local storage. Somewhat slow but can work well depending on the platform.
-
-Each model should define `MI_THEAP_INITASNULL` to signify that the initial value
-returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap).
-This incurs an extra check in the fast path (but can often be combined in an existing check).
-------------------------------------------------------------------------------------------- */
-
-static inline mi_theap_t* _mi_theap_default(void);
-static inline mi_theap_t* _mi_theap_cached(void);
-
-#if defined(_WIN32)
-  #define MI_TLS_MODEL_DYNAMIC_WIN32        1    
-#elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__)  // macOS on arm64 or x64
-  // #define MI_TLS_MODEL_DYNAMIC_PTHREADS  1    // also works but a bit slower
-  #define MI_TLS_MODEL_FIXED_SLOT           1
-  #define MI_TLS_MODEL_FIXED_SLOT_DEFAULT   108  // seems unused. @apple: it would be great to get 2 official slots for custom allocators :-)
-  #define MI_TLS_MODEL_FIXED_SLOT_CACHED    109
-  // we used before __PTK_FRAMEWORK_OLDGC_KEY9 (89) but that seems used now.
-  // see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
-#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__)
-  #define MI_TLS_MODEL_DYNAMIC_PTHREADS     1
-  // #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL  1
-#else
-  #define MI_TLS_MODEL_THREAD_LOCAL         1
-#endif
-
-// Declared this way to optimize register spills and branches
-mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void);
-
-static inline mi_theap_t* __mi_theap_empty(void) {
-  #if __GNUC__
-  __asm("");  // prevent conditional load
-  return (mi_theap_t*)&_mi_theap_empty;
-  #else
-  return _mi_theap_empty_get();
-  #endif
-}
-
-#if MI_TLS_MODEL_THREAD_LOCAL
-// Thread local with an initial value (default on Linux). Very efficient.
-
-extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default;  // default theap to allocate from
-extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached;   // theap from the last used heap
-
-static inline mi_theap_t* _mi_theap_default(void) {
-  #if defined(MI_TLS_RECURSE_GUARD)
-  if (mi_unlikely(!_mi_process_is_initialized)) return _mi_theap_empty_get();
-  #endif
-  return __mi_theap_default;
-}
-
-static inline mi_theap_t* _mi_theap_cached(void) {
-  return __mi_theap_cached;
-}
-
-#elif MI_TLS_MODEL_FIXED_SLOT
-// Fixed TLS slot (default on macOS).
-#define MI_THEAP_INITASNULL  1
-
-static inline mi_theap_t* _mi_theap_default(void) {
-  return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_DEFAULT);
-}
-
-static inline mi_theap_t* _mi_theap_cached(void) {
-  return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_CACHED);
-}
-
-#elif MI_TLS_MODEL_DYNAMIC_WIN32
-// Dynamic TLS slot (default on Windows)
-#define MI_THEAP_INITASNULL  1
-
-// We try to use direct slots (64), but can also use the expansion slots (upto 1024 extra available)
-// See <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm> for the offsets.
-#if MI_SIZE_SIZE==4
-#define MI_TLS_EXPANSION_SLOT    (0x0F94 / MI_SIZE_SIZE)
-#else
-#define MI_TLS_EXPANSION_SLOT    (0x1780 / MI_SIZE_SIZE)
-#endif
-
-extern mi_decl_hidden size_t _mi_theap_default_slot;
-extern mi_decl_hidden size_t _mi_theap_cached_slot;
-extern mi_decl_hidden size_t _mi_theap_default_expansion_slot;
-extern mi_decl_hidden size_t _mi_theap_cached_expansion_slot;
-
-static inline mi_theap_t* _mi_theap_default(void) {
-  const size_t slot = _mi_theap_default_slot;
-  mi_theap_t* theap  = (mi_theap_t*)mi_prim_tls_slot(slot);
-  #if !MI_WIN_DIRECT_TLS
-  if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
-    if mi_likely(theap!=NULL) {                 // initialized (on this thread)?
-      theap = ((mi_theap_t**)theap)[_mi_theap_default_expansion_slot];
-    }
-  }
-  #endif
-  return theap;
-}
-
-static inline mi_theap_t* _mi_theap_cached(void) {
-  const size_t slot = _mi_theap_cached_slot;
-  mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
-  #if !MI_WIN_DIRECT_TLS
-  if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
-    if mi_likely(theap!=NULL) {                 // initialized (on this thread)?
-      theap = ((mi_theap_t**)theap)[_mi_theap_cached_expansion_slot];
-    }
-  }
-  #endif
-  return theap;
-}
-
-#elif MI_TLS_MODEL_DYNAMIC_PTHREADS
-// Dynamic pthread slot on less common platforms. This is not too bad. (default on OpenBSD)
-#define MI_THEAP_INITASNULL  1
-
-extern mi_decl_hidden pthread_key_t _mi_theap_default_key;
-extern mi_decl_hidden pthread_key_t _mi_theap_cached_key;
-
-static inline mi_theap_t* _mi_theap_default(void) {
-  #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
-  // we can skip this check if using the initial key will return NULL from pthread_getspecific
-  if mi_unlikely(_mi_theap_default_key==0) { return NULL; }
-  #endif
-  return (mi_theap_t*)pthread_getspecific(_mi_theap_default_key);
-}
-
-static inline mi_theap_t* _mi_theap_cached(void) {
-  #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
-  // we can skip this check if using the initial key will return NULL from pthread_getspecific
-  if mi_unlikely(_mi_theap_cached_key==0) { return NULL; }
-  #endif
-  return (mi_theap_t*)pthread_getspecific(_mi_theap_cached_key);
-}
-
-#else
-#error "no TLS model is defined for this platform?"
-#endif
-
-
-// Check if a thread is initialized (without using a thread-local if using fixed slots)
-static inline bool _mi_thread_is_initialized(void) {
-  return (mi_theap_is_initialized(_mi_theap_default()));
-}
-
-// Get (and possible create) the theap belonging to a heap
-// We cache the last accessed theap in `_mi_theap_cached` for better performance.
-static inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) {
-  mi_theap_t* theap = _mi_theap_cached();  
-  #if MI_THEAP_INITASNULL
-  if mi_likely(theap!=NULL && _mi_theap_heap(theap)==heap) return theap;
-  #else
-  if mi_likely(_mi_theap_heap(theap)==heap) return theap;
-  #endif
-  return _mi_heap_theap_get_or_init(heap);
-}
-
-// Get the theap belonging to a heap without creating in if it is not yet initialized.
-static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {
-  mi_theap_t* theap = _mi_theap_cached();
-  #if MI_THEAP_INITASNULL
-  if mi_unlikely(theap==NULL || _mi_theap_heap(theap)!=heap)
-  #else
-  if mi_unlikely(_mi_theap_heap(theap)!=heap)
-  #endif
-  {
-    theap = _mi_heap_theap_get_peek(heap);  // don't update the cache on a query (?)
-  }
-  mi_assert(theap==NULL || _mi_theap_heap(theap)==heap);
-  return theap;
-}
-
-// Find the associated theap or NULL if it does not exist (during shutdown)
-// Should be fast as it is called in `free.c:mi_free_try_collect`.
-static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) {
-  mi_heap_t* const heap = page->heap;
-  mi_theap_t* theap;
-  if mi_likely(heap==NULL) { theap = __mi_theap_main; }  // note: on macOS accessing the thread_local can cause allocation during thread shutdown (and reinitialize the thread)!
-                      else { theap = _mi_heap_theap_peek(heap); }
-  mi_assert_internal(theap==NULL || _mi_thread_id()==theap->tld->thread_id);
-  return theap;
-}
-
-#endif  // MI_PRIM_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/track.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/track.h
deleted file mode 100644
index 1f402129..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/track.h
+++ /dev/null
@@ -1,150 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MI_TRACK_H
-#define MI_TRACK_H
-
-/* ------------------------------------------------------------------------------------------------------
-Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers.
-These can be defined for tracking allocation:
-
-  #define mi_track_malloc_size(p,reqsize,size,zero)
-  #define mi_track_free_size(p,_size)
-
-The macros are set up such that the size passed to `mi_track_free_size`
-always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`).
-The `reqsize` is what the user requested, and `size >= reqsize`.
-The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled,
-or otherwise it is the usable block size which may be larger than the original request.
-Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc).
-The `zero` parameter is `true` if the allocated block is zero initialized.
-
-Optional:
-
-  #define mi_track_align(p,alignedp,offset,size)
-  #define mi_track_resize(p,oldsize,newsize)
-  #define mi_track_init()
-
-The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block.
-The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`).
-The `mi_track_resize` is currently unused but could be called on reallocations within a block.
-`mi_track_init` is called at program start.
-
-The following macros are for tools like asan and valgrind to track whether memory is
-defined, undefined, or not accessible at all:
-
-  #define mi_track_mem_defined(p,size)
-  #define mi_track_mem_undefined(p,size)
-  #define mi_track_mem_noaccess(p,size)
-
--------------------------------------------------------------------------------------------------------*/
-
-#if MI_TRACK_VALGRIND
-// valgrind tool
-
-#define MI_TRACK_ENABLED      1
-#define MI_TRACK_HEAP_DESTROY 1           // track free of individual blocks on theap_destroy
-#define MI_TRACK_TOOL         "valgrind"
-
-#include <valgrind/valgrind.h>
-#include <valgrind/memcheck.h>
-
-#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero)
-#define mi_track_free_size(p,_size)               VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/)
-#define mi_track_resize(p,oldsize,newsize)        VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/)
-#define mi_track_mem_defined(p,size)              VALGRIND_MAKE_MEM_DEFINED(p,size)
-#define mi_track_mem_undefined(p,size)            VALGRIND_MAKE_MEM_UNDEFINED(p,size)
-#define mi_track_mem_noaccess(p,size)             VALGRIND_MAKE_MEM_NOACCESS(p,size)
-
-#elif MI_TRACK_ASAN
-// address sanitizer
-
-#define MI_TRACK_ENABLED      1
-#define MI_TRACK_HEAP_DESTROY 0
-#define MI_TRACK_TOOL         "asan"
-
-#include <sanitizer/asan_interface.h>
-
-#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size)
-#define mi_track_free_size(p,size)                ASAN_POISON_MEMORY_REGION(p,size)
-#define mi_track_mem_defined(p,size)              ASAN_UNPOISON_MEMORY_REGION(p,size)
-#define mi_track_mem_undefined(p,size)            ASAN_UNPOISON_MEMORY_REGION(p,size)
-#define mi_track_mem_noaccess(p,size)             ASAN_POISON_MEMORY_REGION(p,size)
-
-#elif MI_TRACK_ETW
-// windows event tracing
-
-#define MI_TRACK_ENABLED      1
-#define MI_TRACK_HEAP_DESTROY 1
-#define MI_TRACK_TOOL         "ETW"
-
-#include "../src/prim/windows/etw.h"
-
-#define mi_track_init()                           EventRegistermicrosoft_windows_mimalloc()
-#define mi_track_done()                           EventUnregistermicrosoft_windows_mimalloc()
-#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size)
-#define mi_track_free_size(p,size)                EventWriteETW_MI_FREE((UINT64)(p), size)
-
-#else
-// no tracking
-
-#define MI_TRACK_ENABLED      0
-#define MI_TRACK_HEAP_DESTROY 0
-#define MI_TRACK_TOOL         "none"
-
-#define mi_track_malloc_size(p,reqsize,size,zero)
-#define mi_track_free_size(p,_size)
-
-#endif
-
-// -------------------
-// Utility definitions
-
-#ifndef mi_track_resize
-#define mi_track_resize(p,oldsize,newsize)      do{ mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false); } while(0)
-#endif
-
-#ifndef mi_track_align
-#define mi_track_align(p,alignedp,offset,size)  mi_track_mem_noaccess(p,offset)
-#endif
-
-#ifndef mi_track_init
-#define mi_track_init()
-#endif
-
-#ifndef mi_track_done
-#define mi_track_done()
-#endif
-
-#ifndef mi_track_mem_defined
-#define mi_track_mem_defined(p,size)
-#endif
-
-#ifndef mi_track_mem_undefined
-#define mi_track_mem_undefined(p,size)
-#endif
-
-#ifndef mi_track_mem_noaccess
-#define mi_track_mem_noaccess(p,size)
-#endif
-
-
-#if MI_PADDING
-#define mi_track_malloc(p,reqsize,zero) \
-  if ((p)!=NULL) { \
-    mi_assert_internal(mi_usable_size(p)==(reqsize)); \
-    mi_track_malloc_size(p,reqsize,reqsize,zero); \
-  }
-#else
-#define mi_track_malloc(p,reqsize,zero) \
-  if ((p)!=NULL) { \
-    mi_assert_internal(mi_usable_size(p)>=(reqsize)); \
-    mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \
-  }
-#endif
-
-#endif // MI_TRACK_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/types.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/types.h
deleted file mode 100644
index 31fdace9..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/include/mimalloc/types.h
+++ /dev/null
@@ -1,732 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MI_TYPES_H
-#define MI_TYPES_H
-
-// --------------------------------------------------------------------------
-// This file contains the main type definitions for mimalloc:
-// mi_heap_t      : all data for a heap; usually there is just one main default heap.
-// mi_theap_t     : a thread local heap belonging to a specific heap:
-//                  maintains lists of thread-local heap pages that have free space.
-// mi_page_t      : a mimalloc page (usually 64KiB or 512KiB) from
-//                  where objects of a single size are allocated.
-//                  Note: we write "OS page" for OS memory pages while
-//                  using plain "page" for mimalloc pages (`mi_page_t`).
-// mi_arena_t     : a large memory area where pages are allocated (process shared)
-// mi_tld_t       : thread local data
-// mi_subproc_t   : all heaps belong to a sub-process (usually just the main one)
-// --------------------------------------------------------------------------
-
-
-#include <mimalloc-stats.h>
-#include <stddef.h>   // ptrdiff_t
-#include <stdint.h>   // uintptr_t, uint16_t, etc
-#include <stdbool.h>  // bool
-#include <limits.h>   // SIZE_MAX etc.
-#include <errno.h>    // error codes
-#include "bits.h"     // size defines (MI_INTPTR_SIZE etc), bit operations
-#include "atomic.h"   // _Atomic primitives
-
-// Minimal alignment necessary. On most platforms 16 bytes are needed
-// due to SSE registers for example. This must be at least `sizeof(void*)`
-#ifndef MI_MAX_ALIGN_SIZE
-#define MI_MAX_ALIGN_SIZE  16   // sizeof(max_align_t)
-#endif
-
-
-// ------------------------------------------------------
-// Variants
-// ------------------------------------------------------
-
-// Define NDEBUG in the release version to disable assertions.
-// #define NDEBUG
-
-// Define MI_TRACK_<tool> to enable tracking support
-// #define MI_TRACK_VALGRIND 1
-// #define MI_TRACK_ASAN     1
-// #define MI_TRACK_ETW      1
-
-// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
-// #define MI_STAT 1
-
-// Define MI_SECURE to enable security mitigations
-// #define MI_SECURE 1  // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption
-// #define MI_SECURE 2  // randomize relative allocation addresses (within mimalloc pages)
-// #define MI_SECURE 3  // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
-// #define MI_SECURE 4  // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)
-// #define MI_SECURE 5  // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)
-
-#if !defined(MI_SECURE)
-#define MI_SECURE 0
-#endif
-
-// Define MI_DEBUG for assertion and invariant checking
-// #define MI_DEBUG 1  // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON)
-// #define MI_DEBUG 2  // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON)
-// #define MI_DEBUG 3  // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON)
-#if !defined(MI_DEBUG)
-#if defined(MI_BUILD_RELEASE) || defined(NDEBUG)
-#define MI_DEBUG 0
-#else
-#define MI_DEBUG 2
-#endif
-#endif
-
-// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking)
-#ifndef MI_STAT
-#if (MI_DEBUG>0)
-#define MI_STAT 2
-#else
-#define MI_STAT 0
-#endif
-#endif
-
-// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options)
-#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL 
-#define MI_GUARDED  1
-#endif
-
-// Reserve extra padding at the end of each block to be more resilient against theap block overflows.
-// The padding can detect buffer overflow on free.
-#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))
-#define MI_PADDING  1
-#endif
-
-// Check padding bytes; allows byte-precise buffer overflow detection
-#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1)
-#define MI_PADDING_CHECK 1
-#endif
-
-
-// Encoded free lists allow detection of corrupted free lists
-// and can detect buffer overflows, modify after free, and double `free`s.
-#if (MI_SECURE>=3 || MI_DEBUG>=1)
-#define MI_ENCODE_FREELIST  1
-#endif
-
-// Enable large pages for objects between 64KiB and 512KiB.
-// This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB
-// are quite random which can lead to too many partially used large pages (but see issue #1104).
-#ifndef MI_ENABLE_LARGE_PAGES
-#define MI_ENABLE_LARGE_PAGES  1
-#endif
-
-// Place page meta info at the start of the page area or keep it separate? 
-// Separate keeps the page info at the arena start (default) which is more secure 
-// and reduces wasted space due to alignment and block sizes. 
-// (but also reserves more memory up front (about 2MiB per GiB))
-#if !defined(MI_PAGE_META_IS_SEPARATED)
-#if MI_PAGE_MAP_FLAT
-#define MI_PAGE_META_IS_SEPARATED    0
-#else
-#define MI_PAGE_META_IS_SEPARATED    1
-#endif
-#endif
-
-// We can choose to only put page info of small pages at the start of the page area.
-// This can be used to have a slightly faster `mi_free_small` function for specialized
-// cases (like language runtime systems).
-#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL)
-#define MI_PAGE_META_ALIGNED_FREE_SMALL   0
-#endif
-
-// Configuration checks
-#if !MI_PAGE_META_IS_SEPARATED && MI_SECURE
-#error "secure mode should use separated page infos"
-#endif
-#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE
-#error "secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL"
-#endif
-#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT
-#error "cannot have a flat page map with separated page infos"
-#endif
-#if MI_DEBUG && NDEBUG
-#warning "mimalloc assertions enabled in a release build"
-#endif
-
-
-// --------------------------------------------------------------
-// Sizes of internal data-structures
-// (comments specify sizes on 64-bit, usually 32-bit is halved)
-// --------------------------------------------------------------
-
-// Main size parameter; determines max arena sizes and max arena object sizes etc.
-#ifndef MI_ARENA_SLICE_SHIFT
-  #ifdef  MI_SMALL_PAGE_SHIFT   // backward compatibility
-  #define MI_ARENA_SLICE_SHIFT              MI_SMALL_PAGE_SHIFT
-  #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64
-  #define MI_ARENA_SLICE_SHIFT              (17)                        // 128 KiB to not waste too much due to 16 KiB guard pages
-  #else
-  #define MI_ARENA_SLICE_SHIFT              (13 + MI_SIZE_SHIFT)        // 64 KiB (32 KiB on 32-bit)
-  #endif
-#endif
-#if MI_ARENA_SLICE_SHIFT < 12
-#error Arena slices should be at least 4KiB
-#endif
-
-#ifndef MI_BCHUNK_BITS_SHIFT
-  #if MI_ARENA_SLICE_SHIFT <= 13    // <= 8KiB
-  #define MI_BCHUNK_BITS_SHIFT              (7)   // 128 bits
-  #elif MI_ARENA_SLICE_SHIFT < 16   // <= 32KiB
-  #define MI_BCHUNK_BITS_SHIFT              (8)   // 256 bits
-  #else
-  #define MI_BCHUNK_BITS_SHIFT              (6 + MI_SIZE_SHIFT)       // 512 bits (or 256 on 32-bit)
-  #endif
-#endif
-
-#define MI_BCHUNK_BITS                    (1 << MI_BCHUNK_BITS_SHIFT)         // sub-bitmaps in arena's are "bchunks" of 512 bits
-#define MI_ARENA_SLICE_SIZE               (MI_ZU(1) << MI_ARENA_SLICE_SHIFT)  // arena's allocate in slices of 64 KiB
-#define MI_ARENA_SLICE_ALIGN              (MI_ARENA_SLICE_SIZE)
-
-#define MI_ARENA_MIN_OBJ_SLICES           (1)
-#define MI_ARENA_MAX_CHUNK_OBJ_SLICES     (MI_BCHUNK_BITS)                    // 32 MiB (or 8 MiB on 32-bit)
-
-#define MI_ARENA_MIN_OBJ_SIZE             (MI_ARENA_MIN_OBJ_SLICES * MI_ARENA_SLICE_SIZE)
-#define MI_ARENA_MAX_CHUNK_OBJ_SIZE       (MI_ARENA_MAX_CHUNK_OBJ_SLICES * MI_ARENA_SLICE_SIZE)
-
-#if MI_ARENA_MAX_CHUNK_OBJ_SIZE < MI_SIZE_SIZE*1024
-#error maximum object size may be too small to hold local thread data
-#endif
-
-#define MI_SMALL_PAGE_SIZE                MI_ARENA_MIN_OBJ_SIZE                    // 64 KiB
-#define MI_MEDIUM_PAGE_SIZE               (8*MI_SMALL_PAGE_SIZE)                   // 512 KiB  (=byte in the bchunk bitmap)
-#define MI_LARGE_PAGE_SIZE                (MI_SIZE_SIZE*MI_MEDIUM_PAGE_SIZE)       // 4 MiB    (=word in the bchunk bitmap)
-
-
-// Maximum number of size classes. (spaced exponentially in 12.5% increments)
-#if MI_BIN_HUGE != 73U
-#error "mimalloc internal: expecting 73 bins"
-#endif
-#define MI_BIN_FULL  (MI_BIN_HUGE+1)
-#define MI_BIN_COUNT (MI_BIN_FULL+1)
-
-// We never allocate more than PTRDIFF_MAX (see also <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
-#define MI_MAX_ALLOC_SIZE        PTRDIFF_MAX
-
-// Minimal commit for a page on-demand commit (should be >= OS page size)
-#define MI_PAGE_MIN_COMMIT_SIZE  MI_ARENA_SLICE_SIZE
-
-
-// ------------------------------------------------------
-// Arena's are large reserved areas of memory allocated from
-// the OS that are managed by mimalloc to efficiently
-// allocate MI_ARENA_SLICE_SIZE slices of memory for the
-// mimalloc pages.
-// ------------------------------------------------------
-
-// A large memory arena where pages are allocated in.
-typedef struct mi_arena_s mi_arena_t;     // defined below
-
-
-// ------------------------------------------------------
-// Heaps contain allocated blocks. Heaps are self-contained
-// but share the (sub-process) memory in the arena's.
-// ------------------------------------------------------
-
-// A first-class heap.
-typedef struct mi_heap_s mi_heap_t;       // heaps
-
-// ------------------------------------------------------
-// We can have sub-processes that are fully separated
-// from each other (for running multiple Python interpreters
-// for example). A sub-process holds the memory arenas and heaps.
-// ------------------------------------------------------
-
-// A sub-process
-typedef struct mi_subproc_s mi_subproc_t;
-
-
-// ---------------------------------------------------------------
-// a memory id tracks the provenance of arena/OS allocated memory
-// ---------------------------------------------------------------
-
-// Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated.
-// The memid keeps track of this.
-typedef enum mi_memkind_e {
-  MI_MEM_NONE,      // not allocated
-  MI_MEM_EXTERNAL,  // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example)
-  MI_MEM_STATIC,    // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`))
-  MI_MEM_META,      // allocated with the meta data allocator (`arena-meta.c`)
-  MI_MEM_OS,        // allocated from the OS
-  MI_MEM_OS_HUGE,   // allocated as huge OS pages (usually 1GiB, pinned to physical memory)
-  MI_MEM_OS_REMAP,  // allocated in a remapable area (i.e. using `mremap`)
-  MI_MEM_ARENA,     // allocated from an arena (the usual case) (`arena.c`)
-  MI_MEM_HEAP_MAIN  // allocated in the main heap (for theaps)
-} mi_memkind_t;
-
-static inline bool mi_memkind_is_os(mi_memkind_t memkind) {
-  return (memkind >= MI_MEM_OS && memkind <= MI_MEM_OS_REMAP);
-}
-
-static inline bool mi_memkind_needs_no_free(mi_memkind_t memkind) {
-  return (memkind <= MI_MEM_STATIC);
-}
-
-
-typedef struct mi_memid_os_info {
-  void*         base;               // actual base address of the block (used for offset aligned allocations)
-  size_t        size;               // allocated full size
-  // size_t        alignment;       // alignment at allocation
-} mi_memid_os_info_t;
-
-typedef struct mi_memid_arena_info {
-  mi_arena_t*   arena;              // arena that contains this memory
-  uint32_t      slice_index;        // slice index in the arena
-  uint32_t      slice_count;        // allocated slices
-} mi_memid_arena_info_t;
-
-typedef struct mi_memid_meta_info {
-  void*         meta_page;          // meta-page that contains the block
-  uint32_t      block_index;        // block index in the meta-data page
-  uint32_t      block_count;        // allocated blocks
-} mi_memid_meta_info_t;
-
-typedef struct mi_memid_s {
-  union {
-    mi_memid_os_info_t    os;       // only used for MI_MEM_OS
-    mi_memid_arena_info_t arena;    // only used for MI_MEM_ARENA
-    mi_memid_meta_info_t  meta;     // only used for MI_MEM_META
-  } mem;
-  mi_memkind_t  memkind;
-  bool          is_pinned;          // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages)
-  bool          initially_committed;// `true` if the memory was originally allocated as committed
-  bool          initially_zero;     // `true` if the memory was originally zero initialized
-} mi_memid_t;
-
-
-static inline bool mi_memid_is_os(mi_memid_t memid) {
-  return mi_memkind_is_os(memid.memkind);
-}
-
-static inline bool mi_memid_needs_no_free(mi_memid_t memid) {
-  return mi_memkind_needs_no_free(memid.memkind);
-}
-
-static inline mi_arena_t* mi_memid_arena(mi_memid_t memid) {
-  return (memid.memkind == MI_MEM_ARENA ? memid.mem.arena.arena : NULL);
-}
-
-
-// ------------------------------------------------------
-// Mimalloc pages contain allocated blocks
-// ------------------------------------------------------
-
-// The free lists use encoded next fields
-// (Only actually encodes when MI_ENCODED_FREELIST is defined.)
-typedef uintptr_t  mi_encoded_t;
-
-// thread id's
-typedef size_t     mi_threadid_t;
-
-// free lists contain blocks
-typedef struct mi_block_s {
-  mi_encoded_t next;
-} mi_block_t;
-
-
-// The page flags are put in the bottom 2 bits of the thread_id (for a fast test in `mi_free`)
-// If `has_interior_pointers` is true if the page has pointers at an offset in a block (so we have to unalign to the block start before free-ing)
-// `in_full_queue` is true if the page is full and resides in the full queue (so we move it to a regular queue on free-ing)
-#define MI_PAGE_IN_FULL_QUEUE           MI_ZU(0x01)
-#define MI_PAGE_HAS_INTERIOR_POINTERS   MI_ZU(0x02)
-#define MI_PAGE_FLAG_MASK               MI_ZU(0x03)
-typedef size_t mi_page_flags_t;
-
-// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue),
-// and 4 for abandoned & mapped threads -- abandoned-mapped pages are abandoned but also mapped
-// in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse.
-// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)
-#define MI_THREADID_ABANDONED           MI_ZU(0)
-#define MI_THREADID_ABANDONED_MAPPED    (MI_PAGE_FLAG_MASK + 1)
-
-// Thread free list.
-// Points to a list of blocks that are freed by other threads.
-// The least-bit is set if the page is owned by the current thread. (`mi_page_is_owned`).
-// Ownership is required before we can read any non-atomic fields in the page.
-// This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`.
-typedef uintptr_t mi_thread_free_t;
-
-// A page contains blocks of one specific size (`block_size`).
-// Each page has three list of free blocks:
-// `free` for blocks that can be allocated,
-// `local_free` for freed blocks that are not yet available to `mi_malloc`
-// `thread_free` for freed blocks by other threads
-// The `local_free` and `thread_free` lists are migrated to the `free` list
-// when it is exhausted. The separate `local_free` list is necessary to
-// implement a monotonic heartbeat. The `thread_free` list is needed for
-// avoiding atomic operations when allocating from the owning thread.
-//
-// `used - |thread_free|` == actual blocks that are in use (alive)
-// `used - |thread_free| + |free| + |local_free| == capacity`
-//
-// We don't count "freed" (as |free|) but use only the `used` field to reduce
-// the number of memory accesses in the `mi_page_all_free` function(s).
-// Use `_mi_page_free_collect` to collect the thread_free list and update the `used` count.
-//
-// Notes:
-// - Non-atomic fields can only be accessed if having _ownership_ (low bit of `xthread_free` is 1).
-//   Combining the `thread_free` list with an ownership bit allows a concurrent `free` to atomically
-//   free an object and (re)claim ownership if the page was abandoned.
-// - If a page is not part of a theap it is called "abandoned"  (`theap==NULL`) -- in
-//   that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that
-//   are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages).
-// - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`.
-// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`
-// - Using `uint16_t` does not seem to slow things down
-
-typedef struct mi_page_s {
-  _Atomic(mi_threadid_t)    xthread_id;        // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`)
-
-  mi_block_t*               free;              // list of available free blocks (`malloc` allocates from this list)
-  uint16_t                  used;              // number of blocks in use (including blocks in `thread_free`)
-  uint16_t                  capacity;          // number of blocks committed
-  uint16_t                  reserved;          // number of blocks reserved in memory
-  uint8_t                   retire_expire;     // expiration count for retired blocks
-  bool                      free_is_zero;      // `true` if the blocks in the free list are zero initialized
-
-  mi_block_t*               local_free;        // list of deferred free blocks by this thread (migrates to `free`)
-  _Atomic(mi_thread_free_t) xthread_free;      // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`)
-
-  size_t                    block_size;        // const: size available in each block (always `>0`)
-  uint8_t*                  page_start;        // const: start of the blocks
-
-  #if (MI_ENCODE_FREELIST || MI_PADDING)
-  uintptr_t                 keys[2];           // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary
-  #endif
-
-  mi_theap_t*               theap;             // the theap owning this page (may not be valid or NULL for abandoned pages)
-  mi_heap_t*                heap;              // const: the heap owning this page
-
-  struct mi_page_s*         next;              // next page owned by the theap with the same `block_size`
-  struct mi_page_s*         prev;              // previous page owned by the theap with the same `block_size`
-  size_t                    slice_committed;   // committed size relative to the first arena slice of the page data (or 0 if the page is fully committed already)
-  mi_memid_t                memid;             // const: provenance of the page memory
-} mi_page_t;
-
-
-// ------------------------------------------------------
-// Object sizes
-// ------------------------------------------------------
-
-#define MI_PAGE_ALIGN                     MI_ARENA_SLICE_ALIGN      // pages must be aligned on this for the page map.
-#define MI_PAGE_MIN_START_BLOCK_ALIGN     MI_MAX_ALIGN_SIZE         // minimal block alignment for the first block in a page (16b)
-#define MI_PAGE_MAX_START_BLOCK_ALIGN2    (4*MI_KiB)                // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment)
-#define MI_PAGE_OSPAGE_BLOCK_ALIGN2       (4*MI_KiB)                // also aligns any multiple of this size to avoid TLB misses.
-#define MI_PAGE_MAX_OVERALLOC_ALIGN       MI_ARENA_SLICE_SIZE       // (64 KiB) limit for which we overallocate in arena pages, beyond this use OS allocation
-
-// The max object sizes are intended to not waste more than ~ 12.5% internally over the page sizes.
-#define MI_SMALL_MAX_OBJ_SIZE             ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6)   // = 10 KiB
-#if MI_ENABLE_LARGE_PAGES
-#define MI_MEDIUM_MAX_OBJ_SIZE            ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6)  // ~ 84 KiB
-#define MI_LARGE_MAX_OBJ_SIZE             (MI_LARGE_PAGE_SIZE/8)    // <= 512 KiB // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
-#else
-#define MI_MEDIUM_MAX_OBJ_SIZE            (MI_MEDIUM_PAGE_SIZE/8)   // <= 64 KiB
-#define MI_LARGE_MAX_OBJ_SIZE             MI_MEDIUM_MAX_OBJ_SIZE    // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
-#endif
-#define MI_LARGE_MAX_OBJ_WSIZE            (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE)
-
-#if (MI_LARGE_MAX_OBJ_WSIZE >= 655360)
-#error "mimalloc internal: define more bins"
-#endif
-
-
-// ------------------------------------------------------
-// Page kinds
-// ------------------------------------------------------
-
-typedef enum mi_page_kind_e {
-  MI_PAGE_SMALL,      // small blocks go into 64KiB pages
-  MI_PAGE_MEDIUM,     // medium blocks go into 512KiB pages
-  MI_PAGE_LARGE,      // larger blocks go into 4MiB pages (if `MI_ENABLE_LARGE_PAGES==1`)
-  MI_PAGE_SINGLETON   // page containing a single block.
-                      // used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an alignment `> MI_PAGE_MAX_OVERALLOC_ALIGN`.
-} mi_page_kind_t;
-
-
-
-// ------------------------------------------------------
-// A "theap" is a thread local heap which owns pages.
-// (making them thread-local avoids atomic operations)
-//
-// All theaps belong to a (non-thread-local) heap.
-// A theap just owns a set of pages for allocation and
-// can only be allocate/reallocate from the thread that created it.
-// Freeing blocks can be done from any thread though.
-//
-// Per thread, there is always a default theap that belongs
-// to the default heap. It is initialized to statically
-// point initially to an empty theap to avoid initialization
-// checks in the fast path.
-// ------------------------------------------------------
-
-// Thread local data
-typedef struct mi_tld_s mi_tld_t;   // defined below
-
-// Pages of a certain block size are held in a queue.
-typedef struct mi_page_queue_s {
-  mi_page_t* first;
-  mi_page_t* last;
-  size_t     count;
-  size_t     block_size;
-} mi_page_queue_t;
-
-// Random context
-typedef struct mi_random_cxt_s {
-  uint32_t input[16];
-  uint32_t output[16];
-  int      output_available;
-  bool     weak;
-} mi_random_ctx_t;
-
-
-// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows
-#if MI_PADDING
-typedef struct mi_padding_s {
-  uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
-  uint32_t delta;  // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
-} mi_padding_t;
-#define MI_PADDING_SIZE   (sizeof(mi_padding_t))
-#define MI_PADDING_WSIZE  ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE)
-#else
-#define MI_PADDING_SIZE   0
-#define MI_PADDING_WSIZE  0
-#endif
-
-#define MI_PAGES_DIRECT   (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)
-
-
-// A thread-local heap ("theap") owns a set of thread-local pages.
-struct mi_theap_s {
-  mi_tld_t*             tld;                                 // thread-local data
-  _Atomic(mi_heap_t*)   heap;                                // the heap this theap belongs to.
-  _Atomic(size_t)       refcount;                            // reference count
-  unsigned long long    heartbeat;                           // monotonic heartbeat count
-  uintptr_t             cookie;                              // random cookie to verify pointers (see `_mi_ptr_cookie`)
-  mi_random_ctx_t       random;                              // random number context used for secure allocation
-  size_t                page_count;                          // total number of pages in the `pages` queues.
-  size_t                page_retired_min;                    // smallest retired index (retired pages are fully free, but still in the page queues)
-  size_t                page_retired_max;                    // largest retired index into the `pages` array.
-  size_t                pages_full_size;                     // optimization: total size of blocks in the pages of the full queue (issue #1220)
-  long                  generic_count;                       // how often is `_mi_malloc_generic` called?
-  long                  generic_collect_count;               // how often is `_mi_malloc_generic` called without collecting?
-
-  mi_theap_t*           tnext;                               // list of theaps in this thread
-  mi_theap_t*           tprev;  
-  mi_theap_t*           hnext;                               // list of theaps of the owning `heap`
-  mi_theap_t*           hprev;
-  
-  long                  page_full_retain;                    // how many full pages can be retained per queue (before abandoning them)
-  bool                  allow_page_reclaim;                  // `true` if this theap should not reclaim abandoned pages
-  bool                  allow_page_abandon;                  // `true` if this theap can abandon pages to reduce memory footprint
-  #if MI_GUARDED
-  size_t                guarded_size_min;                    // minimal size for guarded objects
-  size_t                guarded_size_max;                    // maximal size for guarded objects
-  size_t                guarded_sample_rate;                 // sample rate (set to 0 to disable guarded pages)
-  size_t                guarded_sample_count;                // current sample count (counting down to 0)
-  #endif
-  mi_page_t*            pages_free_direct[MI_PAGES_DIRECT];  // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
-  mi_page_queue_t       pages[MI_BIN_COUNT];                 // queue of pages for each size class (or "bin")
-  mi_memid_t            memid;                               // provenance of the theap struct itself (meta or os)
-  mi_stats_t            stats;                               // thread-local statistics
-};
-
-
-
-
-// ------------------------------------------------------
-// Heaps contain allocated blocks. Heaps are self-contained
-// but share the (sub-process) memory in the arena's.
-// ------------------------------------------------------
-
-// Keep track of all owned and abandoned pages in the arena's
-struct mi_arena_pages_s;
-typedef struct mi_arena_pages_s mi_arena_pages_t;
-
-#define MI_MAX_ARENAS   (160)   // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `mi_arena_reserve`)
-                                // 160 arenas is enough for ~2 TiB memory
-
-// A dynamic thread-local variable; 0 for an invalid thread-local
-typedef size_t mi_thread_local_t;
-
-typedef struct mi_heap_s {
-  mi_subproc_t*         subproc;                        // a heap belongs to a subprocess
-  size_t                heap_seq;                       // unique sequence number for heaps in this subprocess
-  mi_heap_t*            next;                           // list of heaps in this subprocess
-  mi_heap_t*            prev;
-  mi_thread_local_t     theap;                          // dynamic thread local for the thread-local theaps of this heap
-
-  mi_arena_t*           exclusive_arena;                // if the heap should only allocate from a specific arena (or NULL)
-  int                   numa_node;                      // if >=0, prefer this numa node for allocations
-
-  mi_theap_t*           theaps;                         // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields)
-  mi_lock_t             theaps_lock;                    // lock for the theaps list operations
-
-  _Atomic(size_t)       abandoned_count[MI_BIN_COUNT];  // total count of abandoned pages in this heap
-  mi_page_t*            os_abandoned_pages;             // list of pages that are OS allocated and not in an arena
-  mi_lock_t             os_abandoned_pages_lock;        // lock for the os abandoned pages list (this lock protects list operations)
-
-  _Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL)
-  mi_lock_t             arena_pages_lock;                // lock to update the arena_pages array
-
-  mi_stats_t            stats;                           // statistics for this heap; periodically updated by merging from each theap
-} mi_heap_t;
-
-
-// ------------------------------------------------------
-// Sub processes do not reclaim or visit pages from other sub processes.
-// These are essentially the static variables of a process, and
-// usually there is only one subprocess. This can be used for example
-// by CPython to have separate interpreters within one process.
-// Each thread can only belong to one subprocess
-// (and needs to call `mi_subproc_add_current_thread` before any allocations).
-// ------------------------------------------------------
-
-struct mi_subproc_s {
-  size_t                subproc_seq;                    // unique id for sub-processes
-  mi_subproc_t*         next;                           // list of all sub-processes
-  mi_subproc_t*         prev;
-
-  _Atomic(size_t)       arena_count;                    // current count of arena's
-  _Atomic(mi_arena_t*)  arenas[MI_MAX_ARENAS];          // arena's of this sub-process
-  mi_lock_t             arena_reserve_lock;             // lock to ensure arena's get reserved one at a time
-  mi_decl_align(8)                                      // needed on some 32-bit platforms
-  _Atomic(int64_t)      purge_expire;                   // expiration is set if any arenas can be purged
-
-  _Atomic(mi_heap_t*)   heap_main;                      // main heap for this sub process
-  mi_heap_t*            heaps;                          // heaps belonging to this sub-process
-  mi_lock_t             heaps_lock;
-
-  _Atomic(size_t)       thread_count;                   // current threads associated with this sub-process
-  _Atomic(size_t)       thread_total_count;             // total created threads associated with this sub-process
-  _Atomic(size_t)       heap_count;                     // current heaps in this sub-process (== |heaps|)
-  _Atomic(size_t)       heap_total_count;               // total created heaps in this sub-process
-
-  mi_memid_t            memid;                          // provenance of this memory block (meta or static)
-  mi_decl_align(8)                                      // needed on some 32-bit platforms
-  mi_stats_t            stats;                          // subprocess statistics; updated for arena/OS stats like committed,
-                                                        // and otherwise merged with heap stats when those are deleted
-};
-
-
-// ------------------------------------------------------
-// Thread Local data
-// ------------------------------------------------------
-
-// Milliseconds as in `int64_t` to avoid overflows
-typedef int64_t  mi_msecs_t;
-
-// Thread local data
-struct mi_tld_s {
-  mi_threadid_t         thread_id;            // thread id of this thread
-  size_t                thread_seq;           // thread sequence id (linear count of created threads)
-  int                   numa_node;            // thread preferred numa node
-  mi_subproc_t*         subproc;              // sub-process this thread belongs to.
-  mi_theap_t*           theaps;               // list of theaps in this thread (so we can abandon all when the thread terminates)
-  mi_lock_t             theaps_lock;          // lock as the theaps list is sometimes accessed from another thread (on `mi_heap_free`)
-  bool                  recurse;              // true if deferred was called; used to prevent infinite recursion.
-  bool                  is_in_threadpool;     // true if this thread is part of a threadpool (and can run arbitrary tasks)
-  mi_memid_t            memid;                // provenance of the tld memory itself (meta or OS)
-};
-
-
-/* ----------------------------------------------------------------------------
-  Arenas are fixed area's of OS memory from which we can allocate
-  large blocks (>= MI_ARENA_MIN_BLOCK_SIZE).
-  In contrast to the rest of mimalloc, the arenas are shared between
-  threads and need to be accessed using atomic operations (using atomic `mi_bitmap_t`'s).
-
-  Arenas are also used to for huge OS page (1GiB) reservations or for reserving
-  OS memory upfront which can be improve performance or is sometimes needed
-  on embedded devices. We can also employ this with WASI or `sbrk` systems
-  to reserve large arenas upfront and be able to reuse the memory more effectively.
------------------------------------------------------------------------------*/
-
-#define MI_ARENA_BIN_COUNT      (MI_BIN_COUNT)
-#define MI_ARENA_MIN_SIZE       (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE)           // 32 MiB (or 8 MiB on 32-bit)
-#define MI_ARENA_MAX_SIZE       (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE)
-
-typedef struct mi_bitmap_s  mi_bitmap_t;    // atomic bitmap  (defined in `src/bitmap.h`)
-typedef struct mi_bbitmap_s mi_bbitmap_t;   // atomic binned bitmap (defined in `src/bitmap.h`)
-
-typedef struct mi_arena_pages_s {
-  mi_bitmap_t* pages;                // all registered pages (abandoned and owned)
-  mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT];  // abandoned pages per size bin (a set bit means the start of the page)
-  // followed by the bitmaps (whose siz`es depend on the arena size)
-} mi_arena_pages_t;
-
-
-// A memory arena
-typedef struct mi_arena_s {
-  mi_memid_t          memid;                // provenance of the memory area
-  mi_subproc_t*       subproc;              // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`)
-  size_t              arena_idx;            // index in the arenas array
-
-  size_t              slice_count;          // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)
-  size_t              info_slices;          // initial slices reserved for the arena bitmaps
-  int                 numa_node;            // associated NUMA node
-  bool                is_exclusive;         // only allow allocations if specifically for this arena
-  mi_decl_align(8)                          // needed on some 32-bit platforms
-  _Atomic(mi_msecs_t) purge_expire;         // expiration time when slices can be purged from `slices_purge`.
-  mi_commit_fun_t*    commit_fun;           // custom commit/decommit memory
-  void*               commit_fun_arg;       // user argument for a custom commit function
-
-  size_t              total_size;           // for (user given) memory more than MI_ARENA_MAX_SIZE, we use N arena's to cover it. The first (parent) has the total size (and the other sub-arena's 0).
-  mi_arena_t*         parent;               // if this is a sub arena, this points to the first one in the memory area.
-
-  mi_bbitmap_t*       slices_free;          // is the slice free? (a binned bitmap with size classes)
-  mi_bitmap_t*        slices_committed;     // is the slice committed? (i.e. accessible)
-  mi_bitmap_t*        slices_dirty;         // is the slice potentially non-zero?
-  mi_bitmap_t*        slices_purge;         // slices that can be purged
-  mi_page_t*          pages_meta;           // pre-allocated `slice_count` page meta info -- only used if `MI_PAGE_META_IS_SEPARATED!=0`
-  mi_arena_pages_t    pages_main;           // arena page bitmaps for the main heap are allocated up front as well
-
-  // followed by the bitmaps (whose sizes depend on the arena size)
-  // note: when adding bitmaps revise `mi_arena_info_slices_needed`
-} mi_arena_t;
-
-
-
-/* -----------------------------------------------------------
-  Error codes passed to `_mi_fatal_error`
-  All are recoverable but EFAULT is a serious error and aborts by default in secure mode.
-  For portability define undefined error codes using common Unix codes:
-  <https://www-numi.fnal.gov/offline_software/srt_public_context/WebDocs/Errors/unix_system_errors.html>
------------------------------------------------------------ */
-
-#ifndef EAGAIN         // double free
-#define EAGAIN (11)
-#endif
-#ifndef ENOMEM         // out of memory
-#define ENOMEM (12)
-#endif
-#ifndef EFAULT         // corrupted free-list or meta-data
-#define EFAULT (14)
-#endif
-#ifndef EINVAL         // trying to free an invalid pointer
-#define EINVAL (22)
-#endif
-#ifndef EOVERFLOW      // count*size overflow
-#define EOVERFLOW (75)
-#endif
-
-/* -----------------------------------------------------------
-  Debug constants
------------------------------------------------------------ */
-
-#if !defined(MI_DEBUG_UNINIT)
-#define MI_DEBUG_UNINIT     (0xD0)
-#endif
-#if !defined(MI_DEBUG_FREED)
-#define MI_DEBUG_FREED      (0xDF)
-#endif
-#if !defined(MI_DEBUG_PADDING)
-#define MI_DEBUG_PADDING    (0xDE)
-#endif
-
-
-#endif // MI_TYPES_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-aligned.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-aligned.c
deleted file mode 100644
index 97dc6045..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-aligned.c
+++ /dev/null
@@ -1,441 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"  // _mi_theap_default
-
-#include <string.h>     // memset
-
-// ------------------------------------------------------
-// Aligned Allocation
-// ------------------------------------------------------
-
-static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {
-  // certain blocks are always allocated at a certain natural alignment.
-  // (see also `arena.c:mi_arenas_page_alloc_fresh`).
-  mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0));
-  if (alignment > size) return false;
-  const size_t bsize = mi_good_size(size);
-  const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) ||             // power-of-two under N
-                  (alignment==MI_PAGE_OSPAGE_BLOCK_ALIGN2 && (bsize % MI_PAGE_OSPAGE_BLOCK_ALIGN2)==0);  // or multiple of N
-  if (ok) { mi_assert_internal((bsize & (alignment-1)) == 0); } // since both power of 2 and alignment <= size
-  return ok;
-}
-
-#if MI_GUARDED
-static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero) mi_attr_noexcept {
-  // use over allocation for guarded blocksl
-  #if MI_THEAP_INITASNULL
-  if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }
-  #endif
-  mi_assert_internal(alignment > 0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN);
-  if mi_unlikely(alignment >= MI_PAGE_MAX_OVERALLOC_ALIGN || size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE - alignment)) {
-    _mi_error_message(EOVERFLOW, "(guarded) aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
-    return NULL;
-  }
-  const size_t oversize = size + alignment - 1;
-  void* const base = _mi_theap_malloc_guarded(theap, oversize, zero);
-  if (base==NULL) return NULL;
-  void* const p = _mi_align_up_ptr(base, alignment);
-  mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size);
-  mi_assert_internal(mi_usable_size(p) >= size);
-  mi_assert_internal(_mi_is_aligned(p, alignment));
-  return p;
-}
-
-static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {
-  #if MI_THEAP_INITASNULL
-  if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }
-  #endif
-  const size_t rate = theap->guarded_sample_rate;
-  // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty`
-  if (rate != 0) { theap->guarded_sample_rate = 0; }
-  void* p = _mi_theap_malloc_zero(theap, size, zero, usable);
-  if (rate != 0) { theap->guarded_sample_rate = rate; }
-  return p;
-}
-#else
-static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {
-  return _mi_theap_malloc_zero(theap, size, zero, usable);
-}
-#endif
-
-// Fallback aligned allocation that over-allocates -- split out for better codegen
-static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
-{
-  mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE));
-  mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
-
-  void* p;
-  size_t oversize;
-  if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
-    // use OS allocation for large alignments and allocate inside a singleton page (not in an arena)
-    // This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned
-    // in the first (and single) page such that the page info is `MI_PAGE_ALIGN` bytes before it (and can be found in the _mi_page_map).
-    if mi_unlikely(offset != 0) {
-      // todo: cannot support offset alignment for very large alignments yet
-      _mi_error_message(EOVERFLOW, "aligned allocation with a large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset);
-      return NULL;
-    }
-    oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size);
-    // note: no guarded as alignment > 0
-    p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, usable); // the page block size should be large enough to align in the single huge page block
-    if (p == NULL) return NULL;
-  }
-  else {
-    // otherwise over-allocate
-    mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN);
-    mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow
-    oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1;  // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that.
-    p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, usable);
-    if (p == NULL) return NULL;
-  }
-
-  // .. and align within the allocation
-  const uintptr_t align_mask = alignment - 1;  // for any x, `(x & align_mask) == (x % alignment)`
-  const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask;
-  const uintptr_t adjust  = (poffset == 0 ? 0 : alignment - poffset);
-  mi_assert_internal(adjust < alignment);
-  void* aligned_p = (void*)((uintptr_t)p + adjust);
-
-  // note: after the above allocation, the page may be abandoned now (as it became full, see `page.c:_mi_malloc_generic`)
-  // and we no longer own it. We should be careful to only read constant fields in the page,
-  // or use safe atomic access as in `mi_page_set_has_interior_pointers`.
-  // (we can access the page though since the just allocated pointer keeps it alive)
-  mi_page_t* page = _mi_ptr_page(p);
-  if (aligned_p != p) {
-    mi_page_set_has_interior_pointers(page, true);
-    #if MI_GUARDED
-    // set tag to aligned so mi_usable_size works with guard pages
-    if (adjust >= sizeof(mi_block_t)) {
-      mi_block_t* const block = (mi_block_t*)p;
-      block->next = MI_BLOCK_TAG_ALIGNED;
-    }
-    #endif
-    _mi_padding_shrink(page, (mi_block_t*)p, adjust + size);
-  }
-  // todo: expand padding if overallocated ?
-
-  mi_assert_internal(mi_page_usable_block_size(page) >= adjust + size);
-  mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);
-  mi_assert_internal(mi_usable_size(aligned_p)>=size);
-  mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust);
-  #if MI_DEBUG > 1
-  mi_page_t* const apage = _mi_ptr_page(aligned_p);
-  void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);
-  mi_assert_internal(p == unalign_p);
-  #endif
-
-  // now zero the block if needed
-  //if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
-  //  // for the tracker, on huge aligned allocations only from the start of the large block is defined
-  //  mi_track_mem_undefined(aligned_p, size);
-  //  if (zero) {
-  //    _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p));
-  //  }
-  //}
-
-  if (p != aligned_p) {
-    mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p));
-    #if MI_GUARDED
-    mi_track_mem_defined(p, sizeof(mi_block_t));
-    #endif
-  }
-  return aligned_p;
-}
-
-// Generic primitive aligned allocation -- split out for better codegen
-static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
-{
-  mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
-  // we don't allocate more than MI_MAX_ALLOC_SIZE (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
-  if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {
-    _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
-    return NULL;
-  }
-
-  // use regular allocation if it is guaranteed to fit the alignment constraints.
-  // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist
-  // a page with the right block size, and if we always use the over-alloc fallback that would never happen.
-  if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) {
-    void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, usable);
-    mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
-    const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;
-    if mi_likely(is_aligned_or_null) {
-      return p;
-    }
-    else {
-      // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..
-      mi_assert(false);
-      mi_free(p);
-    }
-  }
-
-  // fall back to over-allocation
-  return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,usable);
-}
-
-
-// Primitive aligned allocation
-static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
-{
-  // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size.
-  if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
-    #if MI_DEBUG > 0
-    _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment);
-    #endif
-    return NULL;
-  }
-
-  #if MI_GUARDED
-  #if MI_THEAP_INITASNULL
-  if mi_likely(theap!=NULL)
-  #endif
-  if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) {
-    return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero);
-  }
-  #endif
-
-  // try first if there happens to be a small block available with just the right alignment
-  // since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already
-  // naturally aligned this can be often the case.
-  #if MI_THEAP_INITASNULL
-  if mi_likely(theap!=NULL)
-  #endif
-  {
-    if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) {
-      const uintptr_t align_mask = alignment-1;       // for any x, `(x & align_mask) == (x % alignment)`
-      const size_t padsize = size + MI_PADDING_SIZE;
-      mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize);
-      if mi_likely(page->free != NULL) {
-        const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;
-        if mi_likely(is_aligned)
-        {
-          if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
-          void* p = _mi_page_malloc_zero(theap, page, padsize, zero);
-          mi_assert_internal(p != NULL);
-          mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
-          mi_track_malloc(p, size, zero);
-          return p;
-        }
-      }
-    }
-  }
-
-  // fallback to generic aligned allocation
-  return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, usable);
-}
-
-
-// ------------------------------------------------------
-// Internal mi_theap_malloc_aligned / mi_malloc_aligned
-// ------------------------------------------------------
-
-static mi_decl_restrict void* mi_theap_malloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, false, NULL);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_malloc_aligned_at(theap, size, alignment, 0);
-}
-
-static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL);
-}
-
-static mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_zalloc_aligned_at(theap, size, alignment, 0);
-}
-
-static mi_decl_restrict void* mi_theap_calloc_aligned_at(mi_theap_t* theap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(count, size, &total)) return NULL;
-  return mi_theap_zalloc_aligned_at(theap, total, alignment, offset);
-}
-
-static mi_decl_restrict void* mi_theap_calloc_aligned(mi_theap_t* theap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_calloc_aligned_at(theap, count, size, alignment, 0);
-}
-
-
-// ------------------------------------------------------
-// Aligned Allocation
-// ------------------------------------------------------
-
-mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_malloc_aligned_at(_mi_theap_default(), size, alignment, offset);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_malloc_aligned(_mi_theap_default(), size, alignment);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {
-  return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, block_size);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_zalloc_aligned_at(_mi_theap_default(), size, alignment, offset);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_zalloc_aligned(_mi_theap_default(), size, alignment);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {
-  return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, block_size);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_calloc_aligned_at(_mi_theap_default(), count, size, alignment, offset);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_calloc_aligned(_mi_theap_default(), count, size, alignment);
-}
-
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_malloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_malloc_aligned(_mi_heap_theap(heap), size, alignment);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_zalloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_zalloc_aligned(_mi_heap_theap(heap), size, alignment);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_calloc_aligned_at(_mi_heap_theap(heap), count, size, alignment, offset);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_calloc_aligned(_mi_heap_theap(heap), count, size, alignment);
-}
-
-
-// ------------------------------------------------------
-// Aligned re-allocation
-// ------------------------------------------------------
-
-static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {
-  mi_assert(alignment > 0);
-  if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
-  if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL);
-  size_t size = mi_usable_size(p);
-  if (newsize <= size && newsize >= (size - (size / 2))
-      && (((uintptr_t)p + offset) % alignment) == 0) {
-    return p;  // reallocation still fits, is aligned and not more than 25% waste
-  }
-  else {
-    // note: we don't zero allocate upfront so we only zero initialize the expanded part
-    void* newp = mi_theap_malloc_aligned_at(theap,newsize,alignment,offset);
-    if (newp != NULL) {
-      if (zero && newsize > size) {
-        // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
-        size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
-        _mi_memzero((uint8_t*)newp + start, newsize - start);
-      }
-      _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
-      mi_free(p); // only free if successful
-    }
-    return newp;
-  }
-}
-
-static void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept {
-  mi_assert(alignment > 0);
-  if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
-  return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero);
-}
-
-static void* mi_theap_realloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,offset,false);
-}
-
-static void* mi_theap_realloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
-  return mi_theap_realloc_zero_aligned(theap,p,newsize,alignment,false);
-}
-
-static void* mi_theap_rezalloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_realloc_zero_aligned_at(theap, p, newsize, alignment, offset, true);
-}
-
-static void* mi_theap_rezalloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
-  return mi_theap_realloc_zero_aligned(theap, p, newsize, alignment, true);
-}
-
-static void* mi_theap_recalloc_aligned_at(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(newcount, size, &total)) return NULL;
-  return mi_theap_rezalloc_aligned_at(theap, p, total, alignment, offset);
-}
-
-static void* mi_theap_recalloc_aligned(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(newcount, size, &total)) return NULL;
-  return mi_theap_rezalloc_aligned(theap, p, total, alignment);
-}
-
-
-mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_realloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset);
-}
-
-mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
-  return mi_theap_realloc_aligned(_mi_theap_default(), p, newsize, alignment);
-}
-
-mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_rezalloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset);
-}
-
-mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
-  return mi_theap_rezalloc_aligned(_mi_theap_default(), p, newsize, alignment);
-}
-
-mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_recalloc_aligned_at(_mi_theap_default(), p, newcount, size, alignment, offset);
-}
-
-mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_recalloc_aligned(_mi_theap_default(), p, newcount, size, alignment);
-}
-
-
-mi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_realloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset);
-}
-
-mi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
-  return mi_theap_realloc_aligned(_mi_heap_theap(heap), p, newsize, alignment);
-}
-
-mi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_rezalloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset);
-}
-
-mi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
-  return mi_theap_rezalloc_aligned(_mi_heap_theap(heap), p, newsize, alignment);
-}
-
-mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
-  return mi_theap_recalloc_aligned_at(_mi_heap_theap(heap), p, newcount, size, alignment, offset);
-}
-
-mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
-  return mi_theap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment);
-}
-
-
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-override.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-override.c
deleted file mode 100644
index 93d066dc..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-override.c
+++ /dev/null
@@ -1,396 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-#if !defined(MI_IN_ALLOC_C)
-#error "this file should be included from 'alloc.c' (so aliases can work)"
-#endif
-
-
-#if defined(MI_MALLOC_OVERRIDE) && !defined(_DLL)
-
-#if defined(__APPLE__)
-#include <AvailabilityMacros.h>
-mi_decl_externc void   vfree(void* p);
-mi_decl_externc size_t malloc_size(const void* p);
-mi_decl_externc size_t malloc_good_size(size_t size);
-#endif
-
-// helper definition for C override of C++ new
-typedef void* mi_nothrow_t;
-
-// ------------------------------------------------------
-// Override system malloc
-// ------------------------------------------------------
-
-#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !MI_TRACK_ENABLED
-  // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions
-  #if (defined(__GNUC__) && __GNUC__ >= 9)
-    #pragma GCC diagnostic ignored "-Wattributes"  // or we get warnings that nodiscard is ignored on a forward
-    #define MI_FORWARD(fun)      __attribute__((alias(#fun), used, visibility("default"), copy(fun)));
-  #else
-    #define MI_FORWARD(fun)      __attribute__((alias(#fun), used, visibility("default")));
-  #endif
-  #define MI_FORWARD1(fun,x)      MI_FORWARD(fun)
-  #define MI_FORWARD2(fun,x,y)    MI_FORWARD(fun)
-  #define MI_FORWARD3(fun,x,y,z)  MI_FORWARD(fun)
-  #define MI_FORWARD0(fun,x)      MI_FORWARD(fun)
-  #define MI_FORWARD02(fun,x,y)   MI_FORWARD(fun)
-#else
-  // otherwise use forwarding by calling our `mi_` function
-  #define MI_FORWARD1(fun,x)      { return fun(x); }
-  #define MI_FORWARD2(fun,x,y)    { return fun(x,y); }
-  #define MI_FORWARD3(fun,x,y,z)  { return fun(x,y,z); }
-  #define MI_FORWARD0(fun,x)      { fun(x); }
-  #define MI_FORWARD02(fun,x,y)   { fun(x,y); }
-#endif
-
-#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE)
-  // define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for
-  // functions that are interposed (or the interposing does not work)
-  #define MI_OSX_IS_INTERPOSED
-
-  mi_decl_externc size_t mi_malloc_size_checked(void *p) {
-    if (!mi_is_in_heap_region(p)) return 0;
-    return mi_usable_size(p);
-  }
-
-  // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`
-  // See: <https://books.google.com/books?id=K8vUkpOXhN4C&pg=PA73>
-  struct mi_interpose_s {
-    const void* replacement;
-    const void* target;
-  };
-  #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }
-  #define MI_INTERPOSE_MI(fun)            MI_INTERPOSE_FUN(fun,mi_##fun)
-
-  #define MI_INTERPOSE_DECLS(name)        __attribute__((used)) static struct mi_interpose_s name[]  __attribute__((section("__DATA, __interpose")))
-
-  MI_INTERPOSE_DECLS(_mi_interposes) =
-  {
-    MI_INTERPOSE_MI(malloc),
-    MI_INTERPOSE_MI(calloc),
-    MI_INTERPOSE_MI(realloc),
-    MI_INTERPOSE_MI(strdup),
-    MI_INTERPOSE_MI(realpath),
-    MI_INTERPOSE_MI(posix_memalign),
-    MI_INTERPOSE_MI(reallocf),
-    MI_INTERPOSE_MI(valloc),
-    MI_INTERPOSE_FUN(malloc_size,mi_malloc_size_checked),
-    MI_INTERPOSE_MI(malloc_good_size),
-    #ifdef MI_OSX_ZONE
-    // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely
-    MI_INTERPOSE_MI(free),
-    MI_INTERPOSE_FUN(vfree,mi_free),
-    #else
-    // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity <https://github.com/nneonneo/osx-10.9-opensource/blob/master/objc4-551.1/runtime/hashtable2.mm>)
-    MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us
-    MI_INTERPOSE_FUN(vfree,mi_cfree),
-    #endif
-  };
-  #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
-  MI_INTERPOSE_DECLS(_mi_interposes_10_7)  = { MI_INTERPOSE_MI(strndup) };
-  #endif
-  #if defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15)
-  MI_INTERPOSE_DECLS(_mi_interposes_10_15) = { MI_INTERPOSE_MI(aligned_alloc) };
-  #endif
-
-  #ifdef __cplusplus
-  extern "C" {
-  #endif
-  void  _ZdlPv(void* p);   // delete
-  void  _ZdaPv(void* p);   // delete[]
-  void  _ZdlPvm(void* p, size_t n);  // delete
-  void  _ZdaPvm(void* p, size_t n);  // delete[]
-  void* _Znwm(size_t n);  // new
-  void* _Znam(size_t n);  // new[]
-  void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow
-  void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow
-  #ifdef __cplusplus
-  }
-  #endif
-  __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[]  __attribute__((section("__DATA, __interpose"))) =
-  {
-    MI_INTERPOSE_FUN(_ZdlPv,mi_free),
-    MI_INTERPOSE_FUN(_ZdaPv,mi_free),
-    MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size),
-    MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size),
-    MI_INTERPOSE_FUN(_Znwm,mi_new),
-    MI_INTERPOSE_FUN(_Znam,mi_new),
-    MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow),
-    MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow),
-  };
-
-#elif defined(_MSC_VER)
-  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE
-  void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
-    return mi_expand(_Block, _Size);
-  }
-  _Check_return_ _ACRTIMP 
-  size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT {
-    return mi_malloc_size(_Block);
-  }
-  _Check_return_ _ACRTIMP _CRT_HYBRIDPATCHABLE 
-  size_t __cdecl _msize(_Pre_notnull_ void* _Block) {
-    return mi_malloc_size(_Block);
-  }
-  _ACRTIMP 
-  void __cdecl _free_base(_Pre_maybenull_ _Post_invalid_ void* _Block) {
-    mi_free(_Block);
-  }
-  _ACRTIMP _CRT_HYBRIDPATCHABLE 
-  void __cdecl free(_Pre_maybenull_ _Post_invalid_ void* _Block) {
-    mi_free(_Block);
-  }
-  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_JIT_INTRINSIC _CRTRESTRICT _CRT_HYBRIDPATCHABLE
-  void* __cdecl malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size) {
-    return mi_malloc(_Size);
-  }
-  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _malloc_base(_In_ size_t _Size) {
-    return mi_malloc(_Size);
-  }
-  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _realloc_base(_Pre_maybenull_ _Post_invalid_  void*  _Block, _In_ size_t _Size) {
-    return mi_realloc(_Block, _Size);
-  }
-  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT _CRT_HYBRIDPATCHABLE
-  void* __cdecl realloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW  size_t _Size) {
-    return mi_realloc(_Block, _Size);
-  }
-  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT 
-  void* __cdecl _calloc_base(_In_ size_t _Count, _In_ size_t _Size) {
-    return mi_calloc(_Count, _Size);
-  }
-  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRT_JIT_INTRINSIC _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl calloc(_In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
-    return mi_calloc(_Count, _Size);
-  }
-  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _recalloc_base(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ size_t _Count, _In_ size_t _Size) {
-    return mi_recalloc(_Block, _Count, _Size);
-  }
-  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _recalloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
-    return mi_recalloc(_Block, _Count, _Size);
-  }
-  _ACRTIMP 
-  void __cdecl _aligned_free(_Pre_maybenull_ _Post_invalid_ void* _Block) {
-    mi_free(_Block);
-  }
-  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _aligned_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {
-    return mi_malloc_aligned(_Size, _Alignment);
-  }
-  _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _aligned_offset_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {
-    return mi_malloc_aligned_at(_Size, _Alignment, _Offset);
-  }
-  _Check_return_ _ACRTIMP 
-  size_t __cdecl _aligned_msize(_Pre_notnull_ void*  _Block, _In_ size_t _Alignment, _In_ size_t _Offset) {
-    MI_UNUSED(_Alignment); MI_UNUSED(_Offset); return mi_malloc_size(_Block);
-  }
-  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _aligned_offset_realloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {
-    return mi_realloc_aligned_at(_Block, _Size, _Alignment, _Offset);
-  }
-  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _aligned_offset_recalloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {
-    return mi_recalloc_aligned_at(_Block, _Count, _Size, _Alignment, _Offset);
-  }
-  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _aligned_realloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {
-    return mi_realloc_aligned(_Block, _Size, _Alignment);
-  }
-  _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT
-  void* __cdecl _aligned_recalloc(_Pre_maybenull_ _Post_invalid_ void*  _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {
-    return mi_recalloc_aligned(_Block, _Count, _Size, _Alignment);
-  }
-#else
-  // On all other systems forward allocation primitives to our API
-  mi_decl_export void* malloc(size_t size)              MI_FORWARD1(mi_malloc, size)
-  mi_decl_export void* calloc(size_t size, size_t n)    MI_FORWARD2(mi_calloc, size, n)
-  mi_decl_export void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize)
-  mi_decl_export void  free(void* p)                    MI_FORWARD0(mi_free, p)
-  // In principle we do not need to forward `strdup`/`strndup` but on some systems these do not use `malloc` internally (but a more primitive call)
-  // We only override if `strdup` is not a macro (as on some older libc's, see issue #885)
-  #if !defined(strdup)
-  mi_decl_export char* strdup(const char* str)             MI_FORWARD1(mi_strdup, str)
-  #endif
-  #if !defined(strndup) && (!defined(__APPLE__) || (defined(MAC_OS_X_VERSION_10_7) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7))
-  mi_decl_export char* strndup(const char* str, size_t n)  MI_FORWARD2(mi_strndup, str, n)
-  #endif
-#endif
-
-#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
-#pragma GCC visibility push(default)
-#endif
-
-// ------------------------------------------------------
-// Override new/delete
-// This is not really necessary as they usually call
-// malloc/free anyway, but it improves performance.
-// ------------------------------------------------------
-#ifdef __cplusplus
-  // ------------------------------------------------------
-  // With a C++ compiler we override the new/delete operators.
-  // see <https://en.cppreference.com/w/cpp/memory/new/operator_new>
-  // ------------------------------------------------------
-  #include <new>
-
-  #ifndef MI_OSX_IS_INTERPOSED
-    void operator delete(void* p) noexcept              MI_FORWARD0(mi_free,p)
-    void operator delete[](void* p) noexcept            MI_FORWARD0(mi_free,p)
-
-    void* operator new(std::size_t n) noexcept(false)   MI_FORWARD1(mi_new,n)
-    void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)
-
-    void* operator new  (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }
-    void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }
-
-    #if (__cplusplus >= 201402L || _MSC_VER >= 1916)
-    void operator delete  (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)
-    void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)
-    #endif
-  #endif
-
-  #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5))
-  void operator delete  (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
-  void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
-  void operator delete  (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
-  void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
-  void operator delete  (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
-  void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
-
-  void* operator new( std::size_t n, std::align_val_t al)   noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
-  void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
-  void* operator new  (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
-  void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
-  #endif
-
-#elif (defined(__GNUC__) || defined(__clang__))
-  // ------------------------------------------------------
-  // Override by defining the mangled C++ names of the operators (as
-  // used by GCC and CLang).
-  // See <https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling>
-  // ------------------------------------------------------
-
-  void _ZdlPv(void* p)            MI_FORWARD0(mi_free,p) // delete
-  void _ZdaPv(void* p)            MI_FORWARD0(mi_free,p) // delete[]
-  void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)
-  void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)
-
-  void _ZdlPvSt11align_val_t(void* p, size_t al)            { mi_free_aligned(p,al); }
-  void _ZdaPvSt11align_val_t(void* p, size_t al)            { mi_free_aligned(p,al); }
-  void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
-  void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
-
-  void _ZdlPvRKSt9nothrow_t(void* p, mi_nothrow_t tag)      { MI_UNUSED(tag); mi_free(p); }  // operator delete(void*, std::nothrow_t const&)
-  void _ZdaPvRKSt9nothrow_t(void* p, mi_nothrow_t tag)      { MI_UNUSED(tag); mi_free(p); }  // operator delete[](void*, std::nothrow_t const&)
-  void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&)
-  void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&)
-
-  #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__))  // pr #1257
-    void* _Znwm(size_t n)                             MI_FORWARD1(mi_new,n)  // new 64-bit
-    void* _Znam(size_t n)                             MI_FORWARD1(mi_new,n)  // new[] 64-bit
-    void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
-    void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
-    void* _ZnwmSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al)
-    void* _ZnamSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al)
-    void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
-    void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
-  #elif (MI_INTPTR_SIZE==4)
-    void* _Znwj(size_t n)                             MI_FORWARD1(mi_new,n)  // new 64-bit
-    void* _Znaj(size_t n)                             MI_FORWARD1(mi_new,n)  // new[] 64-bit
-    void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
-    void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
-    void* _ZnwjSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al)
-    void* _ZnajSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al)
-    void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
-    void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
-  #else
-    #error "define overloads for new/delete for this platform (just for performance, can be skipped)"
-  #endif
-#endif // __cplusplus
-
-// ------------------------------------------------------
-// Further Posix & Unix functions definitions
-// ------------------------------------------------------
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef MI_OSX_IS_INTERPOSED
-  // Forward Posix/Unix calls as well
-  void*  reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize)
-  size_t malloc_size(const void* p)        MI_FORWARD1(mi_usable_size,p)
-  #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__)
-  size_t malloc_usable_size(void *p)       MI_FORWARD1(mi_usable_size,p)
-  #else
-  size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p)
-  #endif
-
-  // No forwarding here due to aliasing/name mangling issues
-  void*  valloc(size_t size)               { return mi_valloc(size); }
-  void   vfree(void* p)                    { mi_free(p); }
-  size_t malloc_good_size(size_t size)     { return mi_malloc_good_size(size); }
-  int    posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); }
-
-  // `aligned_alloc` is only available when __USE_ISOC11 is defined.
-  // Note: it seems __USE_ISOC11 is not defined in musl (and perhaps other libc's) so we only check
-  // for it if using glibc.
-  // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot
-  // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9.
-  // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it
-  // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we  can avoid overriding it ourselves.
-  #if !defined(__GLIBC__) || __USE_ISOC11
-  void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }
-  #endif
-#endif
-
-// no forwarding here due to aliasing/name mangling issues
-void  cfree(void* p)                                    { mi_free(p); }
-void* pvalloc(size_t size)                              { return mi_pvalloc(size); }
-void* memalign(size_t alignment, size_t size)           { return mi_memalign(alignment, size); }
-#if !defined(_WIN32)
-void* _aligned_malloc(size_t size, size_t alignment)    { return mi_malloc_aligned(size,alignment); }
-#endif
-void* reallocarray(void* p, size_t count, size_t size)  { return mi_reallocarray(p, count, size); }
-// some systems define reallocarr so mark it as a weak symbol (#751)
-mi_decl_weak int reallocarr(void* p, size_t count, size_t size)    { return mi_reallocarr(p, count, size); }
-
-#if defined(__wasi__)
-  // forward __libc interface (see PR #667)
-  void* __libc_malloc(size_t size)                      MI_FORWARD1(mi_malloc, size)
-  void* __libc_calloc(size_t count, size_t size)        MI_FORWARD2(mi_calloc, count, size)
-  void* __libc_realloc(void* p, size_t size)            MI_FORWARD2(mi_realloc, p, size)
-  void  __libc_free(void* p)                            MI_FORWARD0(mi_free, p)
-  void* __libc_memalign(size_t alignment, size_t size)  { return mi_memalign(alignment, size); }
-
-#elif defined(__linux__)
-  // forward __libc interface (needed for glibc-based and musl-based Linux distributions)
-  void* __libc_malloc(size_t size)                      MI_FORWARD1(mi_malloc,size)
-  void* __libc_calloc(size_t count, size_t size)        MI_FORWARD2(mi_calloc,count,size)
-  void* __libc_realloc(void* p, size_t size)            MI_FORWARD2(mi_realloc,p,size)
-  void  __libc_free(void* p)                            MI_FORWARD0(mi_free,p)
-  void  __libc_cfree(void* p)                           MI_FORWARD0(mi_free,p)
-
-  void* __libc_valloc(size_t size)                      { return mi_valloc(size); }
-  void* __libc_pvalloc(size_t size)                     { return mi_pvalloc(size); }
-  void* __libc_memalign(size_t alignment, size_t size)  { return mi_memalign(alignment,size); }
-  int   __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); }
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
-#pragma GCC visibility pop
-#endif
-
-#endif // MI_MALLOC_OVERRIDE
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-posix.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-posix.c
deleted file mode 100644
index 60639ff4..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc-posix.c
+++ /dev/null
@@ -1,202 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-// ------------------------------------------------------------------------
-// mi prefixed publi definitions of various Posix, Unix, and C++ functions
-// for convenience and used when overriding these functions.
-// ------------------------------------------------------------------------
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-
-// ------------------------------------------------------
-// Posix & Unix functions definitions
-// ------------------------------------------------------
-
-#include <errno.h>
-#include <string.h>  // memset
-#include <stdlib.h>  // getenv
-
-#ifdef _MSC_VER
-#pragma warning(disable:4996)  // getenv _wgetenv
-#endif
-
-#ifndef EINVAL
-#define EINVAL 22
-#endif
-#ifndef ENOMEM
-#define ENOMEM 12
-#endif
-
-
-mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {
-  // if (!mi_is_in_heap_region(p)) return 0;
-  return mi_usable_size(p);
-}
-
-mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
-  // if (!mi_is_in_heap_region(p)) return 0;
-  return mi_usable_size(p);
-}
-
-mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {
-  return mi_good_size(size);
-}
-
-void mi_cfree(void* p) mi_attr_noexcept {
-  if (mi_is_in_heap_region(p)) {
-    mi_free(p);
-  }
-}
-
-int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {
-  // Note: The spec dictates we should not modify `*p` on an error. (issue#27)
-  // <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
-  if (p == NULL) return EINVAL;
-  if ((alignment % sizeof(void*)) != 0) return EINVAL;                 // natural alignment
-  // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned`
-  if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL;  // not a power of 2
-  void* q = mi_malloc_aligned(size, alignment);
-  if (q==NULL && size != 0) return ENOMEM;
-  mi_assert_internal(_mi_is_aligned(q,alignment));
-  *p = q;
-  return 0;
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
-  void* p = mi_malloc_aligned(size, alignment);
-  mi_assert_internal(_mi_is_aligned(p,alignment));
-  return p;
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept {
-  return mi_memalign( _mi_os_page_size(), size );
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {
-  size_t psize = _mi_os_page_size();
-  if (size >= SIZE_MAX - psize) return NULL; // overflow
-  size_t asize = _mi_align_up(size, psize);
-  return mi_malloc_aligned(asize, psize);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
-  // C11 requires the size to be an integral multiple of the alignment, see <https://en.cppreference.com/w/c/memory/aligned_alloc>.
-  // unfortunately, it turns out quite some programs pass a size that is not an integral multiple so skip this check..
-  /* if mi_unlikely((size & (alignment - 1)) != 0) { // C11 requires alignment>0 && integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
-      #if MI_DEBUG > 0
-      _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment);
-      #endif
-      return NULL;
-    }
-  */
-  // C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned
-  void* p = mi_malloc_aligned(size, alignment);
-  mi_assert_internal(_mi_is_aligned(p,alignment));
-  return p;
-}
-
-mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept {  // BSD <https://man.freebsd.org/cgi/man.cgi?query=reallocarray>
-  size_t total;
-  if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
-    errno = EOVERFLOW;
-    return NULL;
-  }
-  void* newp = mi_realloc(p,total);
-  if (newp==NULL) { errno = ENOMEM; }
-  return newp;
-}
-
-mi_decl_nodiscard int mi_reallocarr( void* ptrp, size_t count, size_t size ) mi_attr_noexcept { // NetBSD <https://man.netbsd.org/reallocarr.3>
-  mi_assert(size != 0);
-  mi_assert(ptrp != NULL);
-  if (ptrp == NULL || size == 0) {
-    return (errno = EINVAL);
-  }
-  size_t total;
-  if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
-    return (errno = EOVERFLOW);
-  }
-  void** op = (void**)ptrp;
-  if (total == 0) {
-    free(*op);
-    *op = NULL;
-    return 0;
-  }
-  else {
-    void* newp = mi_realloc(*op,total);
-    if (newp == NULL) { return (errno = ENOMEM); }
-    *op = newp;
-    return 0;
-  }
-}
-
-void* mi__expand(void* p, size_t newsize) mi_attr_noexcept {  // Microsoft
-  void* res = mi_expand(p, newsize);
-  if (res == NULL) { errno = ENOMEM; }
-  return res;
-}
-
-mi_decl_nodiscard mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept {
-  if (s==NULL) return NULL;
-  size_t wlen;
-  for(wlen = 0; s[wlen] != 0 && wlen < PTRDIFF_MAX; wlen++) { }  // prevent overflow on wlen+1
-  size_t size; 
-  if (mi_mul_overflow(wlen+1, sizeof(wchar_t), &size) || size > PTRDIFF_MAX) return NULL;
-  wchar_t* p = (wchar_t*)mi_malloc(size);
-  if (p != NULL) {
-    _mi_memcpy(p,s,size);
-  }
-  return p;
-}
-
-mi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s)  mi_attr_noexcept {
-  return (unsigned char*)mi_strdup((const char*)s);
-}
-
-int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept {
-  if (size != NULL) *size = 0;
-  if (buf==NULL || name==NULL) return EINVAL;
-  char* p = getenv(name);
-  if (p==NULL) {
-    *buf = NULL;
-  }
-  else {
-    *buf = mi_strdup(p);
-    if (*buf==NULL) return ENOMEM;
-    if (size != NULL) { *size = _mi_strlen(p) + 1; }    // cannot overflow as mi_strdup is limited to PTRDIFF_MAX
-  }
-  return 0;
-}
-
-int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept {
-  if (size != NULL) *size = 0;
-  if (buf==NULL || name==NULL) return EINVAL;  
-#if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP))
-  // not supported
-  *buf = NULL;
-  return EINVAL;
-#else
-  wchar_t* p = (wchar_t*)_wgetenv(name);
-  if (p==NULL) {
-    *buf = NULL;
-  }
-  else {
-    *buf = mi_wcsdup(p);
-    if (*buf==NULL) return ENOMEM;
-    if (size != NULL) { *size = wcslen(p) + 1; }  // cannot overflow as wcsdup is limited to PTRDIFF_MAX
-  }
-  return 0;
-#endif
-}
-
-mi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
-  return mi_recalloc_aligned_at(p, newcount, size, alignment, offset);
-}
-
-mi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
-  return mi_recalloc_aligned(p, newcount, size, alignment);
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc.c
deleted file mode 100644
index aa556bd0..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/alloc.c
+++ /dev/null
@@ -1,885 +0,0 @@
-
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#ifndef _DEFAULT_SOURCE
-#define _DEFAULT_SOURCE   // for realpath() on Linux
-#endif
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/atomic.h"
-#include "mimalloc/prim.h"   // _mi_prim_thread_id()
-
-#include <string.h>      // memset, strlen (for mi_strdup)
-#include <stdlib.h>      // malloc, abort
-
-#define MI_IN_ALLOC_C
-#include "alloc-override.c"
-#include "free.c"
-#undef MI_IN_ALLOC_C
-
-// ------------------------------------------------------
-// Allocation
-// ------------------------------------------------------
-
-// Fast allocation in a page: just pop from the free list.
-// Fall back to generic allocation only if the list is empty.
-// Note: in release mode the (inlined) routine is about 7 instructions with a single test.
-static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept
-{
-  if (page->block_size != 0) { // not the empty theap
-    mi_assert_internal(mi_page_block_size(page) >= size);
-    mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-    mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-  }
-
-  // check the free list
-  mi_block_t* const block = page->free;
-  if mi_unlikely(block == NULL) {
-    return _mi_malloc_generic(theap, size, (zero ? 1 : 0), usable);
-  }
-  mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
-  if (usable != NULL) { *usable = mi_page_usable_block_size(page); };
-
-  // pop from the free list
-  page->free = mi_block_next(page, block);
-  page->used++;
-  mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
-  mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE));
-
-  #if MI_DEBUG>3
-  if (page->free_is_zero && size > sizeof(*block)) {
-    mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block)));
-  }
-  #endif
-
-  // allow use of the block internally
-  // note: when tracking we need to avoid ever touching the MI_PADDING since
-  // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`)
-  const size_t bsize = mi_page_usable_block_size(page);
-  mi_track_mem_undefined(block, bsize);
-
-  #if (MI_STAT>0)
-  if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
-    mi_theap_stat_increase(theap, malloc_normal, bsize);
-    #if (MI_STAT>1)
-    mi_theap_stat_counter_increase(theap, malloc_normal_count, 1);
-    const size_t bin = _mi_bin(bsize);
-    mi_theap_stat_increase(theap, malloc_bins[bin], 1);
-    mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE);
-    #endif
-  }
-  #endif
-
-  // zero the block? note: we need to zero the full block size (issue #63)
-  if mi_likely(!zero) {
-    // #if MI_SECURE
-    block->next = 0;  // don't leak internal data
-    // #endif
-    #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
-      if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); }
-    #endif    
-  }
-  else {
-    if (!page->free_is_zero) {
-      _mi_memzero_aligned(block, bsize);
-    }
-    else {
-      block->next = 0;
-      mi_track_mem_defined(block, bsize);
-    }    
-  }
-
-  #if MI_PADDING // && !MI_TRACK_ENABLED
-    mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize);
-    ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));
-    #if (MI_DEBUG>=2)
-    mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta));
-    #endif
-    mi_track_mem_defined(padding,sizeof(mi_padding_t));  // note: re-enable since mi_page_usable_block_size may set noaccess
-    padding->canary = mi_ptr_encode_canary(page,block,page->keys);
-    padding->delta  = (uint32_t)(delta);
-    #if MI_PADDING_CHECK
-    if (!mi_page_is_huge(page)) {
-      uint8_t* fill = (uint8_t*)padding - delta;
-      const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes
-      for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }
-    }
-    #endif
-  #endif
-
-  return block;
-}
-
-// extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`.
-extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept {
-  return mi_page_malloc_zero(theap, page, size, zero, NULL);
-}
-
-#if MI_GUARDED
-mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;
-#endif
-
-// main allocation primitives for small and generic allocation
-
-// internal small size allocation
-static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept
-{
-  mi_assert(theap != NULL);
-  mi_assert(size <= MI_SMALL_SIZE_MAX);
-  #if MI_DEBUG
-  const uintptr_t tid = _mi_thread_id();
-  mi_assert(theap->tld->thread_id == 0 || theap->tld->thread_id == tid); // theaps are thread local
-  #endif
-  #if (MI_PADDING || MI_GUARDED)
-  if mi_unlikely(size == 0) { size = sizeof(void*); }
-  #endif
-  #if MI_GUARDED
-  if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) {
-    return _mi_theap_malloc_guarded(theap, size, zero);
-  }
-  #endif
-
-  // get page in constant time, and allocate from it
-  mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE);
-  void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, usable);
-  mi_track_malloc(p,size,zero);
-
-  #if MI_DEBUG>3
-  if (p != NULL && zero) {
-    mi_assert_expensive(mi_mem_is_zero(p, size));
-  }
-  #endif
-  return p;
-}
-
-// internal generic allocation
-static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept
-{
-  #if MI_GUARDED
-  #if MI_THEAP_INITASNULL
-  if (theap!=NULL)
-  #endif
-  if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) {
-    return _mi_theap_malloc_guarded(theap, size, zero);
-  }
-  #endif
-  #if !MI_THEAP_INITASNULL
-  mi_assert(theap!=NULL);
-  #endif
-  mi_assert(theap==NULL || theap->tld->thread_id == 0 || theap->tld->thread_id == _mi_thread_id());   // theaps are thread local
-  mi_assert((huge_alignment & 1)==0);
-  void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, usable);  // note: size can overflow but it is detected in malloc_generic
-  mi_track_malloc(p, size, zero);
-
-  #if MI_DEBUG>3
-  if (p != NULL && zero) {
-    mi_assert_expensive(mi_mem_is_zero(p, size));
-  }
-  #endif
-  return p;
-}
-
-// internal small allocation
-static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {
-  #if MI_THEAP_INITASNULL
-  if (theap!=NULL) {
-    return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
-  }
-  else {
-    return mi_theap_malloc_generic(theap, size, zero, 0, usable); // tailcall
-  }
-  #else
-  return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
-  #endif
-}
-
-
-// allocate a small block
-mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {
-  return mi_theap_malloc_small_zero(theap, size, false, NULL);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept {
-  return mi_theap_malloc_small(_mi_theap_default(), size);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
-  return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL);
-}
-
-// The main internal allocation functions
-static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {
-  // fast path for small objects
-  if mi_likely(size <= MI_SMALL_SIZE_MAX) {
-    mi_assert_internal(huge_alignment == 0);
-    return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
-  }
-  else {
-    return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);
-  }
-}
-
-extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {
-  // fast path for small objects
-  #if MI_THEAP_INITASNULL
-  if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX)
-  #else
-  if mi_likely(size <= MI_SMALL_SIZE_MAX)
-  #endif
-  {
-    mi_assert_internal(huge_alignment == 0);
-    return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
-  }
-  else {
-    return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);
-  }
-}
-
-void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {
-  return _mi_theap_malloc_zero_ex(theap, size, zero, 0, usable);
-}
-
-
-// Main allocation functions
-
-mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
-  return _mi_theap_malloc_zero(theap, size, false, NULL);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {
-   return mi_theap_malloc(_mi_theap_default(), size);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
-  return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL);
-}
-
-
-// zero initialized small block
-mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {
-  return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL);
-}
-
-mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {
-  return mi_theap_malloc_small_zero(theap, size, true, NULL);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
-  return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL);
-}
-
-
-mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
-  return _mi_theap_malloc_zero(theap, size, true, NULL);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {
-  return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
-  return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL);
-}
-
-mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(count,size,&total)) return NULL;
-  return mi_theap_zalloc(theap,total);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {
-  return mi_theap_calloc(_mi_theap_default(),count,size);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(count, size, &total)) return NULL;
-  return mi_heap_zalloc(heap, total);
-}
-
-// Return usable size
-mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept {
-  return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, usable);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_theap_umalloc(mi_theap_t* theap, size_t size, size_t* usable) mi_attr_noexcept {
-  return _mi_theap_malloc_zero_ex(theap, size, false, 0, usable);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept {
-  return mi_theap_umalloc(_mi_theap_default(), size, usable);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept {
-  return _mi_theap_malloc_zero_ex(_mi_theap_default(), size, true, 0, usable);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(count,size,&total)) return NULL;
-  return mi_uzalloc(total, usable);
-}
-
-// Uninitialized `calloc`
-static mi_decl_restrict void* mi_theap_mallocn(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(count, size, &total)) return NULL;
-  return mi_theap_malloc(theap, total);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {
-  return mi_theap_mallocn(_mi_theap_default(),count,size);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(count, size, &total)) return NULL;
-  return mi_heap_malloc(heap, total);
-}
-
-
-// Expand (or shrink) in place (or fail)
-void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
-  #if MI_PADDING
-  // we do not shrink/expand with padding enabled
-  MI_UNUSED(p); MI_UNUSED(newsize);
-  return NULL;
-  #else
-  if (p == NULL) return NULL;
-  const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand");
-  const size_t size = _mi_usable_size(p,page);
-  if (newsize > size) return NULL;
-  return p; // it fits
-  #endif
-}
-
-void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {
-  // if p == NULL then behave as malloc.
-  // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).
-  // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)
-  const mi_page_t* page;
-  size_t size;
-  if (p==NULL) {
-    page = NULL;
-    size = 0;
-    if (usable_pre!=NULL) { *usable_pre = 0; }
-  }
-  else {
-    page = mi_validate_ptr_page(p,"mi_realloc");
-    size = _mi_usable_size(p,page);
-    if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); }
-  }
-  if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0  // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
-                  && mi_page_heap(page)==_mi_theap_heap(theap))             // and within the same heap
-  {
-    mi_assert_internal(p!=NULL);
-    // todo: do not track as the usable size is still the same in the free; adjust potential padding?
-    // mi_track_resize(p,size,newsize)
-    // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }
-    if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); }
-    return p;  // reallocation still fits and not more than 50% waste
-  }
-  void* newp = mi_theap_umalloc(theap,newsize,usable_post);
-  if mi_likely(newp != NULL) {
-    if (zero && newsize > size) {
-      // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
-      const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
-      _mi_memzero((uint8_t*)newp + start, newsize - start);
-    }
-    else if (newsize == 0) {
-      ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725)
-    }
-    if mi_likely(p != NULL) {
-      const size_t copysize = (newsize > size ? size : newsize);
-      mi_track_mem_defined(p,copysize);  // _mi_useable_size may be too large for byte precise memory tracking..
-      _mi_memcpy(newp, p, copysize);
-      mi_free(p); // only free the original pointer if successful  // todo: optimize since page is known?
-    }
-  }
-  return newp;
-}
-
-mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
-  return _mi_theap_realloc_zero(theap, p, newsize, false, NULL, NULL);
-}
-
-static void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(count, size, &total)) return NULL;
-  return mi_theap_realloc(theap, p, total);
-}
-
-
-// Reallocate but free `p` on errors
-static void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
-  void* newp = mi_theap_realloc(theap, p, newsize);
-  if (newp==NULL && p!=NULL) mi_free(p);
-  return newp;
-}
-
-static void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
-  return _mi_theap_realloc_zero(theap, p, newsize, true, NULL, NULL);
-}
-
-static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {
-  size_t total;
-  if (mi_count_size_overflow(count, size, &total)) return NULL;
-  return mi_theap_rezalloc(theap, p, total);
-}
-
-
-mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {
-  return mi_theap_realloc(_mi_theap_default(),p,newsize);
-}
-
-mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {
-  return mi_theap_reallocn(_mi_theap_default(),p,count,size);
-}
-
-mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {
-  return _mi_theap_realloc_zero(_mi_theap_default(),p,newsize, false, usable_pre, usable_post);
-}
-
-// Reallocate but free `p` on errors
-mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {
-  return mi_theap_reallocf(_mi_theap_default(),p,newsize);
-}
-
-mi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept {
-  return mi_theap_rezalloc(_mi_theap_default(), p, newsize);
-}
-
-mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept {
-  return mi_theap_recalloc(_mi_theap_default(), p, count, size);
-}
-
-
-mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
-  return mi_theap_realloc(_mi_heap_theap(heap), p, newsize);
-}
-
-mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
-  return mi_theap_reallocn(_mi_heap_theap(heap), p, count, size);
-}
-
-// Reallocate but free `p` on errors
-mi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
-  return mi_theap_reallocf(_mi_heap_theap(heap), p, newsize);
-}
-
-mi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
-  return mi_theap_rezalloc(_mi_heap_theap(heap), p, newsize);
-}
-
-mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
-  return mi_theap_recalloc(_mi_heap_theap(heap), p, count, size);
-}
-
-
-
-// ------------------------------------------------------
-// strdup, strndup, and realpath
-// ------------------------------------------------------
-
-// `strdup` using mi_malloc
-mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* theap, const char* s) mi_attr_noexcept {
-  if (s == NULL) return NULL;
-  size_t len = _mi_strlen(s);
-  if (len > MI_MAX_ALLOC_SIZE - 1) return NULL;  // prevent overflow on len+1
-  char* t = (char*)mi_theap_malloc(theap,len+1);
-  if (t == NULL) return NULL;
-  _mi_memcpy(t, s, len);
-  t[len] = 0;
-  return t;
-}
-
-mi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept {
-  return mi_theap_strdup(_mi_theap_default(), s);
-}
-
-mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept {
-  return mi_theap_strdup(_mi_heap_theap(heap), s);
-}
-
-// `strndup` using mi_malloc
-mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* theap, const char* s, size_t n) mi_attr_noexcept {
-  if (s == NULL) return NULL;
-  const size_t len = _mi_strnlen(s,n);  // len <= n
-  if (len > MI_MAX_ALLOC_SIZE - 1) return NULL;  // prevent overflow on len+1
-  char* t = (char*)mi_theap_malloc(theap, len+1);
-  if (t == NULL) return NULL;
-  _mi_memcpy(t, s, len);
-  t[len] = 0;
-  return t;
-}
-
-mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept {
-  return mi_theap_strndup(_mi_theap_default(),s,n);
-}
-
-mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept {
-  return mi_theap_strndup(_mi_heap_theap(heap), s, n);
-}
-
-#ifndef __wasi__
-// `realpath` using mi_malloc
-#ifdef _WIN32
-#ifndef PATH_MAX
-#define PATH_MAX MAX_PATH
-#endif
-
-mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {
-  // todo: use GetFullPathNameW to allow longer file names
-  char buf[PATH_MAX];
-  DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL);
-  if (res == 0) {
-    errno = GetLastError(); return NULL;
-  }
-  else if (res > PATH_MAX) {
-    errno = EINVAL; return NULL;
-  }
-  else if (resolved_name != NULL) {
-    return resolved_name;
-  }
-  else {
-    return mi_theap_strndup(theap, buf, PATH_MAX);
-  }
-}
-
-#else
-
-#include <unistd.h>  // pathconf
-
-static size_t mi_path_max(void) {
-  static _Atomic(size_t) path_max = 0;
-  size_t pmax = mi_atomic_load_acquire(&path_max);
-  if (pmax == 0) {
-    long m = 0;
-    #ifdef _PC_PATH_MAX
-    m = pathconf("/",_PC_PATH_MAX);
-    #endif
-    if (m <= 0) pmax = 4096;      // guess
-    else if (m < 256) pmax = 256; // at least 256
-    else if (m > 64*1024) pmax = 64*1024;  // at most 64 KiB
-    else pmax = m;
-    size_t expected = 0;
-    mi_atomic_cas_strong_acq_rel(&path_max, &expected, pmax);
-  }
-  return pmax;
-}
-
-char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {
-  if (resolved_name != NULL) {
-    return realpath(fname,resolved_name);
-  }
-  else {
-  /*
-    char* rname = realpath(fname, NULL);
-    if (rname == NULL) return NULL;
-    char* result = mi_heap_strdup(heap, rname);
-    mi_cfree(rname);  // note: may leak the original pointer if allocated internally with the system allocator 
-    // note: with ASAN realpath is intercepted and mi_cfree may leak the returned pointer :-(
-    return result;  
-  */
-    const size_t n  = mi_path_max();
-    char* const buf = (char*)mi_zalloc(n+1);
-    if (buf == NULL) {
-      errno = ENOMEM;
-      return NULL;
-    }
-    char* rname  = realpath(fname,buf);
-    char* result = mi_theap_strndup(theap,rname,n); // ok if `rname==NULL`
-    mi_free(buf);
-    return result;
-  }
-}
-#endif
-
-mi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept {
-  return mi_theap_realpath(_mi_theap_default(),fname,resolved_name);
-}
-
-mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept {
-  return mi_theap_realpath(_mi_heap_theap(heap), fname, resolved_name);
-}
-#endif
-
-/*-------------------------------------------------------
-C++ new and new_aligned
-The standard requires calling into `get_new_handler` and
-throwing the bad_alloc exception on failure. If we compile
-with a C++ compiler we can implement this precisely. If we
-use a C compiler we cannot throw a `bad_alloc` exception
-but we call `exit` instead (i.e. not returning).
--------------------------------------------------------*/
-
-#ifdef __cplusplus
-#include <new>
-static bool mi_try_new_handler(bool nothrow) {
-  #if defined(_MSC_VER) || (__cplusplus >= 201103L)
-    std::new_handler h = std::get_new_handler();
-  #else
-    std::new_handler h = std::set_new_handler();
-    std::set_new_handler(h);
-  #endif
-  if (h==NULL) {
-    _mi_error_message(ENOMEM, "out of memory in 'new'");
-    #if defined(_CPPUNWIND) || defined(__cpp_exceptions)  // exceptions are not always enabled
-    if (!nothrow) {
-      throw std::bad_alloc();
-    }
-    #else
-    MI_UNUSED(nothrow);
-    #endif
-    return false;
-  }
-  else {
-    h();
-    return true;
-  }
-}
-#else
-typedef void (*std_new_handler_t)(void);
-
-#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER)))  // exclude clang-cl, see issue #631
-std_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) {
-  return NULL;
-}
-static std_new_handler_t mi_get_new_handler(void) {
-  return _ZSt15get_new_handlerv();
-}
-#else
-// note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`.
-static std_new_handler_t mi_get_new_handler(void) {
-  return NULL;
-}
-#endif
-
-static bool mi_try_new_handler(bool nothrow) {
-  std_new_handler_t h = mi_get_new_handler();
-  if (h==NULL) {
-    _mi_error_message(ENOMEM, "out of memory in 'new'");
-    if (!nothrow) {
-      abort();  // cannot throw in plain C, use abort
-    }
-    return false;
-  }
-  else {
-    h();
-    return true;
-  }
-}
-#endif
-
-static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) {
-  void* p = NULL;
-  while(p == NULL && mi_try_new_handler(nothrow)) {
-    p = mi_theap_malloc(theap,size);
-  }
-  return p;
-}
-
-static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {
-  return mi_theap_try_new(_mi_theap_default(), size, nothrow);
-}
-
-static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) {
-  return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow);
-}
-
-
-mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {
-  void* p = mi_theap_malloc(theap,size);
-  if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false);
-  return p;
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) {
-  return mi_theap_alloc_new(_mi_theap_default(), size);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) {
-  void* p = mi_heap_malloc(heap, size);
-  if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false);
-  return p;
-}
-
-
-mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {
-  size_t total;
-  if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
-    mi_try_new_handler(false);  // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
-    return NULL;
-  }
-  else {
-    return mi_theap_alloc_new(theap,total);
-  }
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) {
-  return mi_theap_alloc_new_n(_mi_theap_default(), count, size);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) {
-  return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size);
-}
-
-
-mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {
-  void* p = mi_malloc(size);
-  if mi_unlikely(p == NULL) return mi_try_new(size, true);
-  return p;
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) {
-  void* p;
-  do {
-    p = mi_malloc_aligned(size, alignment);
-  }
-  while(p == NULL && mi_try_new_handler(false));
-  return p;
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept {
-  void* p;
-  do {
-    p = mi_malloc_aligned(size, alignment);
-  }
-  while(p == NULL && mi_try_new_handler(true));
-  return p;
-}
-
-mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) {
-  void* q;
-  do {
-    q = mi_realloc(p, newsize);
-  } while (q == NULL && mi_try_new_handler(false));
-  return q;
-}
-
-mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {
-  size_t total;
-  if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) {
-    mi_try_new_handler(false);  // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
-    return NULL;
-  }
-  else {
-    return mi_new_realloc(p, total);
-  }
-}
-
-#if MI_GUARDED
-// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true).
-// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)
-// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`
-
-static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) {
-  // TODO: we can still make padding work by moving it out of the guard page area
-  mi_page_t* const page = _mi_ptr_page(block);
-  mi_page_set_has_interior_pointers(page, true);
-  block->next = MI_BLOCK_TAG_GUARDED;
-
-  // set guard page at the end of the block
-  const size_t block_size = mi_page_block_size(page);  // must use `block_size` to match `mi_free_local`
-  const size_t os_page_size = _mi_os_page_size();
-  mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));
-  if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {
-    // should never happen
-    mi_free(block);
-    return NULL;
-  }
-  uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;
-  // note: the alignment of the guard page relies on blocks being os_page_size aligned which
-  // is ensured in `mi_arena_page_alloc_fresh`.  
-  mi_assert_internal(_mi_is_aligned(block, os_page_size));
-  mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));
-  if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) {
-    const bool ok = _mi_os_protect(guard_page, os_page_size);
-    if mi_unlikely(!ok) {
-      _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size);
-    }
-  }
-  else {
-    _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size);
-  }
-
-  // align pointer just in front of the guard page
-  size_t offset = block_size - os_page_size - obj_size;
-  mi_assert_internal(offset > sizeof(mi_block_t));
-  if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) {
-    // give up to place it right in front of the guard page if the offset is too large for unalignment
-    offset = MI_PAGE_MAX_OVERALLOC_ALIGN;
-  }
-  uint8_t* const p = (uint8_t*)block + offset;
-  mi_assert_internal(p == guard_page - obj_size);
-  mi_track_align(block, p, offset, obj_size);
-  mi_track_mem_defined(block, sizeof(mi_block_t));
-  return p;
-}
-
-mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept
-{
-  // allocate multiple of page size ending in a guard page
-  // ensure minimal alignment requirement?
-  if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) {  // check up front so the `req_size` won't overflow    
-    _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size);
-    return NULL;
-  }
-  const size_t os_page_size = _mi_os_page_size();
-  const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));
-  const size_t bsize    = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);
-  const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);  
-  mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, NULL);
-  if (block==NULL) return NULL;
-  void* const p = mi_block_ptr_set_guarded(block, obj_size);
-  if (p == NULL) return p;
-  if (zero) { 
-    _mi_memzero_aligned(p,obj_size);  // we have to zero here as padding might have written here (if the blocksize > reqsize + os_page_size)
-  }
-
-  // stats
-  mi_track_malloc(p, obj_size, zero);    
-  if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); }
-  mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1);
-  #if MI_STAT>1
-  // adjust stats to only count the allocated size of the block (and not the guard page)
-  mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size);
-  mi_theap_stat_increase(theap, malloc_requested, size);
-  #endif
-  #if MI_DEBUG>3
-  if (zero) {
-    mi_assert_expensive(mi_mem_is_zero(p, size));
-  }
-  #endif
-  return p;
-}
-#endif
-
-// ------------------------------------------------------
-// ensure explicit external inline definitions are emitted!
-// ------------------------------------------------------
-
-#ifdef __cplusplus
-void* _mi_externs[] = {
-  (void*)&_mi_page_malloc_zero,
-  (void*)&_mi_theap_malloc_zero,
-  (void*)&_mi_theap_malloc_zero_ex,
-  (void*)&mi_theap_malloc,
-  (void*)&mi_theap_zalloc,
-  (void*)&mi_theap_malloc_small,
-  (void*)&mi_malloc,
-  (void*)&mi_malloc_small,
-  (void*)&mi_zalloc,
-  (void*)&mi_zalloc_small,
-  (void*)&mi_heap_malloc,
-  (void*)&mi_heap_malloc_small,
-  (void*)&mi_malloc_aligned
-  // (void*)&mi_theap_alloc_new,
-  // (void*)&mi_theap_alloc_new_n
-};
-#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena-meta.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena-meta.c
deleted file mode 100644
index cee38caa..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena-meta.c
+++ /dev/null
@@ -1,179 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2024, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* ----------------------------------------------------------------------------
-  We have a special "mini" allocator just for allocation of meta-data like
-  the theap (`mi_theap_t`) or thread-local data (`mi_tld_t`).
-
-  We reuse the bitmap of the arena's for allocation of 64b blocks inside
-  an arena slice (64KiB).
-  We always ensure that meta data is zero'd (we zero on `free`)
------------------------------------------------------------------------------*/
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "bitmap.h"
-
-/* -----------------------------------------------------------
-  Meta data allocation
------------------------------------------------------------ */
-
-#define MI_META_PAGE_SIZE         MI_ARENA_SLICE_SIZE
-#define MI_META_PAGE_ALIGN        MI_ARENA_SLICE_ALIGN
-
-// large enough such that META_MAX_SIZE > 4k (even on 32-bit)
-#define MI_META_BLOCK_SIZE        (1 << (16 - MI_BCHUNK_BITS_SHIFT))        // 128 on 64-bit
-#define MI_META_BLOCK_ALIGN       MI_META_BLOCK_SIZE
-#define MI_META_BLOCKS_PER_PAGE   (MI_META_PAGE_SIZE / MI_META_BLOCK_SIZE)  // 512
-#define MI_META_MAX_SIZE          (MI_BCHUNK_SIZE * MI_META_BLOCK_SIZE)
-
-#if MI_META_MAX_SIZE <= 4096
-#error "max meta object size should be at least 4KiB"
-#endif
-
-typedef struct mi_meta_page_s  {
-  _Atomic(struct mi_meta_page_s*)  next;    // a linked list of meta-data pages (never released)
-  mi_memid_t                       memid;   // provenance of the meta-page memory itself
-  mi_bbitmap_t                     blocks_free;  // a small bitmap with 1 bit per block.
-} mi_meta_page_t;
-
-static mi_decl_cache_align _Atomic(mi_meta_page_t*)  mi_meta_pages = MI_ATOMIC_VAR_INIT(NULL);
-
-
-#if MI_DEBUG > 1
-static mi_meta_page_t* mi_meta_page_of_ptr(void* p, size_t* block_idx) {
-  mi_meta_page_t* mpage = (mi_meta_page_t*)((uint8_t*)_mi_align_down_ptr(p,MI_META_PAGE_ALIGN) + _mi_os_secure_guard_page_size());
-  if (block_idx != NULL) {
-    *block_idx = ((uint8_t*)p - (uint8_t*)mpage) / MI_META_BLOCK_SIZE;
-  }
-  return mpage;
-}
-#endif
-
-static mi_meta_page_t* mi_meta_page_next( mi_meta_page_t* mpage ) {
-  return mi_atomic_load_ptr_acquire(mi_meta_page_t, &mpage->next);
-}
-
-static void* mi_meta_block_start( mi_meta_page_t* mpage, size_t block_idx ) {
-  mi_assert_internal(_mi_is_aligned((uint8_t*)mpage - _mi_os_secure_guard_page_size(), MI_META_PAGE_ALIGN));
-  mi_assert_internal(block_idx < MI_META_BLOCKS_PER_PAGE);
-  void* p = ((uint8_t*)mpage - _mi_os_secure_guard_page_size() + (block_idx * MI_META_BLOCK_SIZE));
-  mi_assert_internal(mpage == mi_meta_page_of_ptr(p,NULL));
-  return p;
-}
-
-// allocate a fresh meta page and add it to the global list.
-static mi_meta_page_t* mi_meta_page_zalloc(void) {
-  // allocate a fresh arena slice
-  // note: careful with _mi_subproc as it may recurse into mi_tld and meta_page_zalloc again.. (same with _mi_os_numa_node()...)
-  mi_memid_t memid;
-  uint8_t* base = (uint8_t*)_mi_arenas_alloc_aligned(mi_heap_main(), MI_META_PAGE_SIZE, MI_META_PAGE_ALIGN, 0,
-                                                                    true /* commit*/, (MI_SECURE==0) /* allow large? */,
-                                                                    NULL /* req arena */, 0 /* thread_seq */, -1 /* numa node */, &memid);
-  if (base == NULL) return NULL;
-  mi_assert_internal(_mi_is_aligned(base,MI_META_PAGE_ALIGN));
-  if (!memid.initially_zero) {
-    _mi_memzero_aligned(base, MI_ARENA_SLICE_SIZE);
-  }
-
-  // guard pages
-  #if MI_SECURE >= 1
-  _mi_os_secure_guard_page_set_at(base, memid);
-  _mi_os_secure_guard_page_set_before(base + MI_META_PAGE_SIZE, memid);
-  #endif
-
-  // initialize the page and free block bitmap
-  mi_meta_page_t* mpage = (mi_meta_page_t*)(base + _mi_os_secure_guard_page_size());
-  mpage->memid = memid;
-  mi_bbitmap_init(&mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */);
-  const size_t mpage_size  = offsetof(mi_meta_page_t,blocks_free) + mi_bbitmap_size(MI_META_BLOCKS_PER_PAGE, NULL);
-  const size_t info_blocks = _mi_divide_up(mpage_size,MI_META_BLOCK_SIZE);
-  const size_t guard_blocks = _mi_divide_up(_mi_os_secure_guard_page_size(), MI_META_BLOCK_SIZE);
-  mi_assert_internal(info_blocks + 2*guard_blocks < MI_META_BLOCKS_PER_PAGE);
-  mi_bbitmap_unsafe_setN(&mpage->blocks_free, info_blocks + guard_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks - 2*guard_blocks);
-
-  // push atomically in front of the meta page list
-  // (note: there is no ABA issue since we never free meta-pages)
-  mi_meta_page_t* old = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);
-  do {
-    mi_atomic_store_ptr_release(mi_meta_page_t, &mpage->next, old);
-  } while(!mi_atomic_cas_ptr_weak_acq_rel(mi_meta_page_t,&mi_meta_pages,&old,mpage));
-  return mpage;
-}
-
-
-// allocate meta-data
-mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid )
-{
-  mi_assert_internal(pmemid != NULL);
-  size = _mi_align_up(size,MI_META_BLOCK_SIZE);
-  if (size == 0 || size > MI_META_MAX_SIZE) return NULL;
-  const size_t block_count = _mi_divide_up(size,MI_META_BLOCK_SIZE);
-  mi_assert_internal(block_count > 0 && block_count < MI_BCHUNK_BITS);
-  mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);
-  mi_meta_page_t* mpage = mpage0;
-  while (mpage != NULL) {
-    size_t block_idx;
-    if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
-      // found and claimed `block_count` blocks
-      *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
-      return mi_meta_block_start(mpage,block_idx);
-    }
-    else {
-      mpage = mi_meta_page_next(mpage);
-    }
-  }
-  // failed to find space in existing pages
-  if (mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages) != mpage0) {
-    // the page list was updated by another thread in the meantime, retry
-    return _mi_meta_zalloc(size,pmemid);
-  }
-  // otherwise, allocate a fresh metapage and try once more
-  mpage = mi_meta_page_zalloc();
-  if (mpage != NULL) {
-    size_t block_idx;
-    if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
-      // found and claimed `block_count` blocks
-      *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
-      return mi_meta_block_start(mpage,block_idx);
-    }
-  }
-  // if all this failed, allocate from the OS
-  return _mi_os_alloc(size, pmemid);
-}
-
-// free meta-data
-mi_decl_noinline void _mi_meta_free(void* p, size_t size, mi_memid_t memid) {
-  if (p==NULL) return;
-  if (memid.memkind == MI_MEM_META) {
-    mi_assert_internal(_mi_divide_up(size, MI_META_BLOCK_SIZE) == memid.mem.meta.block_count);
-    const size_t block_count = memid.mem.meta.block_count;
-    const size_t block_idx   = memid.mem.meta.block_index;
-    mi_meta_page_t* mpage = (mi_meta_page_t*)memid.mem.meta.meta_page;
-    mi_assert_internal(mi_meta_page_of_ptr(p,NULL) == mpage);
-    mi_assert_internal(block_idx + block_count <= MI_META_BLOCKS_PER_PAGE);
-    mi_assert_internal(mi_bbitmap_is_clearN(&mpage->blocks_free, block_idx, block_count));
-    // we zero on free (and on the initial page allocation) so we don't need a "dirty" map
-    _mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE);
-    mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count);
-  }
-  else {
-    _mi_arenas_free(p,size,memid);
-  }
-}
-
-// used for debug output
-bool _mi_meta_is_meta_page(void* p)
-{
-  mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t, &mi_meta_pages);
-  mi_meta_page_t* mpage = mpage0;
-  while (mpage != NULL) {
-    if ((void*)mpage == p) return true;
-    mpage = mi_meta_page_next(mpage);
-  }
-  return false;
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena.c
deleted file mode 100644
index a5686d55..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/arena.c
+++ /dev/null
@@ -1,2514 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* ----------------------------------------------------------------------------
-"Arenas" are fixed area's of OS memory from which we can allocate
-large blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 64KiB).
-In contrast to the rest of mimalloc, the arenas are shared between
-threads and need to be accessed using atomic operations.
-
-Arenas are also used to for huge OS page (1GiB) reservations or for reserving
-OS memory upfront which can be improve performance or is sometimes needed
-on embedded devices. We can also employ this with WASI or `sbrk` systems
-to reserve large arenas upfront and be able to reuse the memory more effectively.
-
-The arena allocation needs to be thread safe and we use an atomic bitmap to allocate.
------------------------------------------------------------------------------*/
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"
-#include "bitmap.h"
-
-/* -----------------------------------------------------------
-  Arena id's
------------------------------------------------------------ */
-
-mi_arena_id_t _mi_arena_id_none(void) {
-  return NULL;
-}
-
-mi_arena_t* _mi_arena_from_id(mi_arena_id_t id) {
-  mi_arena_t* const arena = (mi_arena_t*)id;
-  mi_assert_internal(arena==NULL || arena->parent==NULL); // id's should never point to sub-arena's
-  return arena;
-}
-
-mi_arena_id_t mi_arena_id_from_arena(mi_arena_t* arena) {
-  mi_assert_internal(arena==NULL || arena->parent==NULL);
-  return (arena==NULL ? _mi_arena_id_none() : (mi_arena_id_t)arena);
-}
-
-
-static bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_t* req_arena) {
-  if (arena == req_arena) return true;                         // they match
-  if (arena == NULL) return false;
-  if (req_arena == NULL && !arena->is_exclusive) return true;  // or the arena is not exclusive, and we didn't request a specific one
-  if (arena->parent != NULL && arena->parent == req_arena) return true;  // sub-arena? (note that req_arena is never a sub arena)
-  return false;
-}
-
-bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena) {
-  if (memid.memkind == MI_MEM_ARENA) {
-    return mi_arena_is_suitable(memid.mem.arena.arena, request_arena);
-  }
-  else {
-    return mi_arena_is_suitable(NULL, request_arena);
-  }
-}
-
-size_t mi_arenas_get_count(mi_subproc_t* subproc) {
-  return mi_atomic_load_relaxed(&subproc->arena_count);
-}
-
-mi_arena_t* mi_arena_from_index(mi_subproc_t* subproc, size_t idx) {
-  mi_assert_internal(idx < mi_arenas_get_count(subproc));
-  return mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[idx]);
-}
-
-static size_t mi_arena_info_slices(mi_arena_t* arena) {
-  return arena->info_slices;
-}
-
-#if MI_DEBUG > 1
-static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page) {
-  mi_assert(arena->arena_idx < MI_MAX_ARENAS);
-  mi_arena_pages_t* arena_pages = heap->arena_pages[arena->arena_idx];
-  return (page->memid.memkind == MI_MEM_ARENA &&
-          page->memid.mem.arena.arena == arena &&
-          arena_pages != NULL &&
-          mi_bitmap_is_setN(arena_pages->pages, page->memid.mem.arena.slice_index, 1));
-}
-#endif
-
-size_t mi_arena_min_alignment(void) {
-  return MI_ARENA_SLICE_ALIGN;
-}
-
-size_t mi_arena_min_size(void) {
-  return MI_ARENA_MIN_SIZE;
-}
-
-static size_t mi_arena_max_object_size(void) {
-  size_t max_size = mi_option_get_size(mi_option_arena_max_object_size);
-  max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE);
-  if (max_size <= MI_ARENA_MIN_OBJ_SIZE) {
-    return MI_ARENA_MIN_OBJ_SIZE;
-  }
-  else if (max_size >= MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE) {  // minus a bchunk to accommodate meta info
-    return (MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE);
-  }
-  else {
-    return max_size;
-  }
-}
-
-mi_decl_nodiscard static bool mi_arena_commit(mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) {
-  if (arena != NULL && arena->commit_fun != NULL) {
-    return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg);
-  }
-  else if (already_committed > 0) {
-    return _mi_os_commit_ex(start, size, is_zero, already_committed);
-  }
-  else {
-    return _mi_os_commit(start, size, is_zero);
-  }
-}
-
-
-
-/* -----------------------------------------------------------
-  Util
------------------------------------------------------------ */
-
-
-// Size of an arena
-static size_t mi_arena_size(mi_arena_t* arena) {
-  return mi_size_of_slices(arena->slice_count);
-}
-
-// Start of the arena memory area
-static uint8_t* mi_arena_start(mi_arena_t* arena) {
-  return ((uint8_t*)arena);
-}
-
-// Start of a slice
-uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) {
-  mi_assert_internal(slice_index < arena->slice_count);
-  return (mi_arena_start(arena) + mi_size_of_slices(slice_index));
-}
-
-mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) {
-  mi_assert_internal(slice_index < arena->slice_count);
-  if (arena->pages_meta != NULL) {
-    mi_page_t* const page = &arena->pages_meta[slice_index];
-    #if MI_PAGE_META_ALIGNED_FREE_SMALL
-    // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0)
-    if (page->block_size>0) return page;
-    #else
-    return page;
-    #endif
-  }
-  // fall through (for MI_PAGE_META_ALIGNED_FREE_SMALL)
-  return (mi_page_t*)mi_arena_slice_start(arena,slice_index);
-}
-
-// Arena area
-void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) {
-  if (size != NULL) *size = 0;
-  mi_arena_t* arena = _mi_arena_from_id(arena_id);
-  if (arena == NULL) return NULL;
-  if (size != NULL) {
-    mi_assert_internal(mi_size_of_slices(arena->slice_count) <= arena->total_size);
-    *size = arena->total_size;
-  }
-  return mi_arena_start(arena);
-}
-
-
-// Create an arena memid
-static mi_memid_t mi_memid_create_arena(mi_arena_t* arena, size_t slice_index, size_t slice_count) {
-  mi_assert_internal(slice_index < UINT32_MAX);
-  mi_assert_internal(slice_count < UINT32_MAX);
-  mi_assert_internal(slice_count > 0);
-  mi_assert_internal(slice_index < arena->slice_count);
-  mi_memid_t memid = _mi_memid_create(MI_MEM_ARENA);
-  memid.mem.arena.arena = arena;
-  memid.mem.arena.slice_index = (uint32_t)slice_index;
-  memid.mem.arena.slice_count = (uint32_t)slice_count;
-  return memid;
-}
-
-// get the arena and slice span
-static mi_arena_t* mi_arena_from_memid(mi_memid_t memid, size_t* slice_index, size_t* slice_count) {
-  mi_assert_internal(memid.memkind == MI_MEM_ARENA);
-  mi_arena_t* arena = memid.mem.arena.arena;
-  if (slice_index!=NULL) { *slice_index = memid.mem.arena.slice_index; }
-  if (slice_count!=NULL) { *slice_count = memid.mem.arena.slice_count; }
-  return arena;
-}
-
-static size_t mi_page_full_size(mi_page_t* page) {
-  if (page->memid.memkind == MI_MEM_ARENA) {
-    return page->memid.mem.arena.slice_count * MI_ARENA_SLICE_SIZE;
-  }
-  else if (mi_memid_is_os(page->memid) || page->memid.memkind == MI_MEM_EXTERNAL) {
-    mi_assert_internal((uint8_t*)page->memid.mem.os.base <= (uint8_t*)page);
-    const ptrdiff_t presize = (uint8_t*)page - (uint8_t*)page->memid.mem.os.base;
-    mi_assert_internal((ptrdiff_t)page->memid.mem.os.size >= presize);
-    return (presize > (ptrdiff_t)page->memid.mem.os.size ? 0 : page->memid.mem.os.size - presize);
-  }
-  else {
-    return 0;
-  }
-}
-
-
-/* -----------------------------------------------------------
-  Arena Allocation
------------------------------------------------------------ */
-
-static mi_decl_noinline void* mi_arena_try_alloc_at(
-  mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid)
-{
-  size_t slice_index;
-  if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL;
-
-  // claimed it!
-  void* p = mi_arena_slice_start(arena, slice_index);
-  *memid = mi_memid_create_arena(arena, slice_index, slice_count);
-  memid->is_pinned = arena->memid.is_pinned;
-
-  // set the dirty bits and track which slices become accessible
-  size_t touched_slices = slice_count;
-  if (arena->memid.initially_zero) {
-    size_t already_dirty = 0;
-    memid->initially_zero = mi_bitmap_setN(arena->slices_dirty, slice_index, slice_count, &already_dirty);
-    mi_assert_internal(already_dirty <= touched_slices);
-    touched_slices -= already_dirty;
-  }
-
-  // set commit state
-  if (commit) {
-    // commit requested, but the range may not be committed as a whole: ensure it is committed now
-    const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count);
-    if (already_committed < slice_count) {
-      // not all committed, try to commit now
-      bool commit_zero = false;
-      if (!mi_arena_commit(arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) {
-        // if the commit fails, release ownership, and return NULL;
-        // note: this does not roll back dirty bits but that is ok.
-        mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);
-        return NULL;
-      }
-      if (commit_zero) {
-        memid->initially_zero = true;
-      }
-
-      // set the commit bits
-      mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL);
-
-      // committed
-      #if MI_DEBUG > 1
-      if (memid->initially_zero) {
-        if (!mi_mem_is_zero(p, mi_size_of_slices(slice_count))) {
-          _mi_error_message(EFAULT, "internal error: arena allocation was not zero-initialized!\n");
-          memid->initially_zero = false;
-        }
-      }
-      #endif
-    }
-    else {
-      // already fully committed.
-      _mi_os_reuse(p, mi_size_of_slices(slice_count));
-      // if the OS has overcommit, and this is the first time we access these pages, then
-      // count the commit now (as at arena reserve we didn't count those commits as these are on-demand)
-      if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) {
-        mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices));
-      }
-    }
-
-    mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
-    memid->initially_committed = true;
-
-    // tool support
-    if (memid->initially_zero) {
-      mi_track_mem_defined(p, slice_count * MI_ARENA_SLICE_SIZE);
-    }
-    else {
-      mi_track_mem_undefined(p, slice_count * MI_ARENA_SLICE_SIZE);
-    }
-  }
-  else {
-    // no need to commit, but check if it is already fully committed
-    memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);
-    if (!memid->initially_committed) {
-      // partly committed.. adjust stats
-      size_t already_committed_count = 0;
-      mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count);
-      mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
-      mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count));
-    }
-  }
-
-  mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
-  if (commit) { mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); }
-  if (commit) { mi_assert_internal(memid->initially_committed); }
-  mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
-
-  return p;
-}
-
-
-static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);
-
-// try to reserve a fresh arena space
-static bool mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_large, mi_arena_id_t* arena_id)
-{
-  const size_t arena_count = mi_arenas_get_count(subproc);
-  if (arena_count > (MI_MAX_ARENAS - 4)) return false;
-
-  // calc reserve
-  size_t arena_reserve = mi_option_get_size(mi_option_arena_reserve);
-  if (arena_reserve == 0) return false;
-
-  if (!_mi_os_has_virtual_reserve()) {
-    arena_reserve = arena_reserve/4;  // be conservative if virtual reserve is not supported (for WASM for example)
-  }
-  arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_SLICE_SIZE);
-
-  if (arena_count >= 1 && arena_count <= 128) {
-    // scale up the arena sizes exponentially every 8 entries
-    const size_t multiplier = (size_t)1 << _mi_clamp(arena_count/8, 0, 16);
-    size_t reserve = 0;
-    if (!mi_mul_overflow(multiplier, arena_reserve, &reserve)) {
-      arena_reserve = reserve;
-    }
-  }
-
-  // try to accommodate the requested size for huge allocations
-  req_size = _mi_align_up(req_size + MI_ARENA_MAX_CHUNK_OBJ_SIZE, MI_ARENA_MAX_CHUNK_OBJ_SIZE); // over-reserve for meta-info
-  if (arena_reserve < req_size) {
-    arena_reserve = req_size;
-  }
-
-  // check arena bounds
-  const size_t min_reserve = MI_ARENA_MIN_SIZE;
-  const size_t max_reserve = MI_ARENA_MAX_SIZE;   // 16 GiB
-  if (arena_reserve < min_reserve) {
-    arena_reserve = min_reserve;
-  }
-  else if (arena_reserve > max_reserve) {
-    arena_reserve = max_reserve;
-  }
-
-  // should be able to at least handle the current allocation size
-  if (arena_reserve < req_size) return false;
-
-  // commit eagerly?
-  bool arena_commit = false;
-  const bool overcommit = _mi_os_has_overcommit();
-  if (mi_option_get(mi_option_arena_eager_commit) == 2) { arena_commit = overcommit || mi_option_is_enabled(mi_option_allow_large_os_pages); }
-  else if (mi_option_get(mi_option_arena_eager_commit) == 1) { arena_commit = true; }
-
-  // on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice
-  // is actually allocated for the first time it will be counted.
-  const bool adjust = (overcommit && arena_commit);
-  if (adjust) { mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve); }
-  // and try to reserve the arena
-  int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id);
-  if (err != 0) {
-    if (adjust) { mi_subproc_stat_adjust_increase( subproc, committed, arena_reserve); } // roll back
-    // failed to allocate: try a smaller size arena as fallback?
-    const size_t small_arena_reserve = 4 * MI_ARENA_MIN_SIZE; // 128 MiB (or 32 MiB on 32-bit)
-    if (arena_reserve > small_arena_reserve && small_arena_reserve > req_size) {
-      // try again
-      if (adjust) { mi_subproc_stat_adjust_decrease(subproc, committed, small_arena_reserve); }
-      err = mi_reserve_os_memory_ex2(subproc, small_arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id);
-      if (err != 0 && adjust) { mi_subproc_stat_adjust_increase( subproc, committed, small_arena_reserve); } // roll back
-    }
-  }
-  return (err==0);
-}
-
-
-
-
-/* -----------------------------------------------------------
-  Arena iteration
------------------------------------------------------------ */
-
-static inline bool mi_arena_is_suitable_ex(mi_arena_t* arena, mi_arena_t* req_arena, bool match_numa, int numa_node, bool allow_pinned) {
-  if (!allow_pinned && arena->memid.is_pinned) return false;
-  if (!mi_arena_is_suitable(arena, req_arena)) return false;
-  if (req_arena == NULL) { // if not specific, check numa affinity
-    const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node);
-    if (match_numa) { if (!numa_suitable) return false; }
-               else { if (numa_suitable)  return false; }
-  }
-  return true;
-}
-
-// determine the start of search; important to keep heaps and threads
-// into their own memory regions to reduce contention.
-static size_t mi_arena_start_idx(mi_heap_t* heap, size_t tseq, size_t arena_cycle) {
-  const size_t hseq   = heap->heap_seq;
-  const size_t hcount = mi_atomic_load_relaxed(&heap->subproc->heap_count);
-  if (arena_cycle <= 1)     return 0;
-  if (hseq==0 || hcount<=1 || arena_cycle > 0x8FF) return (tseq % arena_cycle); // common for single heap programs
-
-  // spread heaps evenly among arena's, and then evenly for threads in their fraction
-  size_t start;
-  mi_assert_internal(arena_cycle <= 0x8FF);             // prevent overflow on 32-bit
-  const size_t frac = (arena_cycle * 256) / hcount;     // fraction in the arena_cycle; at most: arena_cycle * 0x100
-  if (frac==0) {
-    // many heaps (> 256 per arena)
-    start = (hseq % arena_cycle);
-  }
-  else {
-    const size_t hspot = (hseq % hcount);  
-    start = (frac * hspot) / 256;           // (arena_cycle * (hseq % hcount)) / hcount
-    if (frac >= 512) {  // at least 2 arena's per heap?
-      start = start + (tseq % (frac/256));
-    }
-  }
-  mi_assert_internal(start < arena_cycle);
-  return start;
-}
-
-#define mi_forall_arenas(heap, req_arena, tseq, name_arena) { \
-  const size_t _arena_count = mi_arenas_get_count(heap->subproc); \
-  const size_t _arena_cycle = (_arena_count == 0 ? 0 : _arena_count - 1); /* first search the arenas below the last one */ \
-  /* always start searching in the arena's below the max */ \
-  const size_t _start = mi_arena_start_idx(heap,tseq,_arena_cycle); \
-  for (size_t _i = 0; _i < _arena_count; _i++) { \
-    mi_arena_t* name_arena; \
-    if (req_arena != NULL) { \
-      name_arena = req_arena; /* if there is a specific req_arena, only search that one */\
-      if (_i > 0) break;      /* only once */ \
-    } \
-    else { \
-      size_t _idx; \
-      if (_i < _arena_cycle) { \
-        _idx = _i + _start; \
-        if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \
-      } \
-      else { \
-        _idx = _i; /* remaining arena's after the cycle */ \
-      } \
-      name_arena = mi_arena_from_index(heap->subproc,_idx); \
-    } \
-    if (name_arena != NULL) \
-    {
-
-#define mi_forall_arenas_end()  \
-    } \
-  } \
-  }
-
-#define mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, numa_node, allow_large, name_arena) \
-  mi_forall_arenas(heap, req_arena,tseq,name_arena) { \
-    if (mi_arena_is_suitable_ex(name_arena, req_arena, match_numa, numa_node, allow_large)) { \
-
-#define mi_forall_suitable_arenas_end() \
-  }} \
-  mi_forall_arenas_end()
-
-/* -----------------------------------------------------------
-  Arena allocation
------------------------------------------------------------ */
-
-// allocate slices from the arenas
-static mi_decl_noinline void* mi_arenas_try_find_free(
-  mi_heap_t* heap, size_t slice_count, size_t alignment,
-  bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)
-{
-  // mi_assert_internal(slice_count <= mi_slice_count_of_size(MI_ARENA_MAX_CHUNK_OBJ_SIZE));
-  mi_assert(alignment <= MI_ARENA_SLICE_ALIGN);
-  if (alignment > MI_ARENA_SLICE_ALIGN) return NULL;
-
-  // search arena's
-  mi_forall_suitable_arenas(heap, req_arena, tseq, true /* only numa matching */, numa_node, allow_large, arena)
-  {
-    void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid);
-    if (p != NULL) return p;
-  }
-  mi_forall_suitable_arenas_end();
-  if (numa_node < 0) return NULL;
-
-  // search again but now regardless of preferred numa affinity
-  mi_forall_suitable_arenas(heap, req_arena, tseq, false /* numa non-matching now */, numa_node, allow_large, arena)
-  {
-    void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid);
-    if (p != NULL) return p;
-  }
-  mi_forall_suitable_arenas_end();
-  return NULL;
-}
-
-// Allocate slices from the arena's -- potentially allocating a fresh arena
-static mi_decl_noinline void* mi_arenas_try_alloc(
-  mi_heap_t* heap,
-  size_t slice_count, size_t alignment,
-  bool commit, bool allow_large,
-  mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)
-{
-  // mi_assert(slice_count <= MI_ARENA_MAX_CHUNK_OBJ_SLICES);
-  mi_assert(alignment <= MI_ARENA_SLICE_ALIGN);
-  void* p;
-
-  // not too large?
-  if (slice_count * MI_ARENA_SLICE_SIZE > MI_ARENA_MAX_SIZE) return NULL;
-
-  // try to find free slices in the arena's
-  p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid);
-  if (p != NULL) return p;
-
-  // did we need a specific arena?
-  if (req_arena != NULL) return NULL;
-
-  // don't create arena's while preloading (todo: or should we?)
-  if (_mi_preloading()) return NULL;
-
-  // don't create arena's if OS allocation is disallowed
-  if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL;
-
-  // otherwise, try to reserve a new arena -- but one thread at a time.. (todo: allow 2 or 4 to reduce contention?)
-  mi_subproc_t* const subproc = heap->subproc;
-  const size_t arena_count = mi_arenas_get_count(subproc);
-  mi_lock(&subproc->arena_reserve_lock) {
-    if (arena_count == mi_arenas_get_count(subproc)) {
-      // we are the first to enter the lock, reserve a fresh arena
-      mi_arena_id_t arena_id = _mi_arena_id_none();
-      mi_arena_reserve(subproc, mi_size_of_slices(slice_count), allow_large, &arena_id);
-    }
-    else {
-      // another thread already reserved a new arena
-    }
-  }
-  // try once more to allocate in the new arena
-  mi_assert_internal(req_arena == NULL);
-  p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid);
-  if (p != NULL) return p;
-
-  return NULL;
-}
-
-// Allocate from the OS (if allowed)
-static void* mi_arena_os_alloc_aligned(
-  size_t size, size_t alignment, size_t align_offset,
-  bool commit, bool allow_large,
-  mi_arena_id_t req_arena_id, mi_memid_t* memid)
-{
-  // if we cannot use OS allocation, return NULL
-  if (mi_option_is_enabled(mi_option_disallow_os_alloc) || req_arena_id != _mi_arena_id_none()) {
-    errno = ENOMEM;
-    return NULL;
-  }
-
-  if (align_offset > 0) {
-    return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid);
-  }
-  else {
-    return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);
-  }
-}
-
-
-// Allocate large sized memory
-void* _mi_arenas_alloc_aligned( mi_heap_t* heap,
-  size_t size, size_t alignment, size_t align_offset,
-  bool commit, bool allow_large,
-  mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)
-{
-  mi_assert_internal(memid != NULL);
-  mi_assert_internal(size > 0);
-
-  // try to allocate in an arena if the alignment is small enough and the object is not too small (as for theap meta data)
-  if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) &&                // is arena allocation allowed?
-      size >= MI_ARENA_MIN_OBJ_SIZE && size <= mi_arena_max_object_size() &&  // and not too small or too large
-      alignment <= MI_ARENA_SLICE_ALIGN && align_offset == 0)                 // and good alignment
-  {
-    const size_t slice_count = mi_slice_count_of_size(size);
-    void* p = mi_arenas_try_alloc(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid);
-    if (p != NULL) return p;
-  }
-
-  // fall back to the OS
-  void* p = mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena, memid);
-  return p;
-}
-
-void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)
-{
-  return _mi_arenas_alloc_aligned(heap, size, MI_ARENA_SLICE_SIZE, 0, commit, allow_large, req_arena, tseq, numa_node, memid);
-}
-
-
-
-/* -----------------------------------------------------------
-  Arena page allocation
------------------------------------------------------------ */
-
-// release ownership of a page. This may free the page if all blocks were concurrently
-// freed in the meantime. Returns true if the page was freed.
-static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap) {
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);
-  mi_thread_free_t tf_new;
-  mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);
-  do {
-    mi_assert_internal(mi_tf_is_owned(tf_old));
-    while mi_unlikely(mi_tf_block(tf_old) != NULL) {
-      _mi_page_free_collect(page, false);  // update used
-      if (mi_page_all_free(page)) {        // it may become free just before unowning it
-        _mi_arenas_page_unabandon(page, current_theap);
-        _mi_arenas_page_free(page, current_theap);
-        return true;
-      }
-      tf_old = mi_atomic_load_relaxed(&page->xthread_free);
-    }
-    mi_assert_internal(mi_tf_block(tf_old)==NULL);
-    tf_new = mi_tf_create(NULL, false);
-  } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new));
-  return false;
-}
-
-
-static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena, bool* keep_abandoned) {
-  // found an abandoned page of the right size
-  mi_page_t* const page  = mi_arena_page_at_slice(arena, slice_index);
-  // can we claim ownership?
-  if (!mi_page_claim_ownership(page)) {
-    // there was a concurrent free that reclaims this page ..
-    // we need to keep it in the abandoned map as the free will call `mi_arena_page_unabandon`,
-    // and wait for readers (us!) to finish. This is why it is very important to set the abandoned
-    // bit again (or otherwise the unabandon will never stop waiting).
-    *keep_abandoned = true;
-    return false;
-  }
-  else {
-    // yes, we can reclaim it, keep the abandoned map entry clear
-    *keep_abandoned = false;
-    return true;
-  }
-}
-
-// allocate initial arena_pages from the main heap
-static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena);
-
-static mi_arena_pages_t* mi_heap_arena_pages(mi_heap_t* heap, mi_arena_t* arena) {
-  mi_assert_internal(arena!=NULL);
-  mi_assert_internal(heap!=NULL);
-  mi_assert(arena->arena_idx < MI_MAX_ARENAS);
-  return mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]);
-}
-
-static mi_arena_t* mi_page_arena_pages(mi_page_t* page, size_t* slice_index, size_t* slice_count, mi_arena_pages_t** parena_pages) {
-  // todo: maybe store the arena* directly in the page?
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_arena_t* const arena = mi_arena_from_memid(page->memid, slice_index, slice_count);
-  mi_assert_internal(arena != NULL);
-  if (parena_pages != NULL) {
-    mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(mi_page_heap(page), arena);
-    mi_assert_internal(arena_pages != NULL);
-    mi_assert_internal(slice_index==NULL || mi_bitmap_is_set(arena_pages->pages, *slice_index));
-    *parena_pages = arena_pages;
-  }
-  return arena;
-}
-
-static mi_arena_pages_t* mi_heap_ensure_arena_pages(mi_heap_t* heap, mi_arena_t* arena) {
-  mi_assert_internal(arena!=NULL);
-  mi_assert_internal(heap!=NULL);
-  mi_assert(arena->arena_idx < MI_MAX_ARENAS);
-  mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);
-  if (arena_pages==NULL) {
-    mi_lock(&heap->arena_pages_lock) {
-      arena_pages = mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]);
-      if (arena_pages == NULL) {  // still NULL?
-        if (_mi_is_heap_main(heap)) {
-          // the page info for the main heap is always allocated as part of an arena
-          arena_pages = &arena->pages_main;
-        }
-        else {
-          // always allocate the arena pages info from the main heap
-          // todo: allocate into the current arena?
-          arena_pages = mi_arena_pages_alloc(arena);
-        }
-        mi_atomic_store_ptr_release(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx], arena_pages);
-      }
-    }
-  }
-  if (_mi_is_heap_main(heap)) { mi_assert(arena_pages != NULL); }  // can never fail
-  return arena_pages;
-}
-
-static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slice_count, size_t block_size)
-{
-  mi_heap_t* const heap = _mi_theap_heap(theap);
-  const size_t tseq = theap->tld->thread_seq;
-  mi_arena_t* const req_arena = heap->exclusive_arena;
-
-  MI_UNUSED(slice_count);
-  const size_t bin = _mi_bin(block_size);
-  mi_assert_internal(bin < MI_BIN_COUNT);
-
-  // any abandoned in our size class?
-  mi_assert_internal(heap != NULL);
-  if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) {
-    return NULL;
-  }
-
-  // search arena's
-  const bool allow_large = true;
-  const int  any_numa = -1;
-  const bool match_numa = true;
-  mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, arena)
-  {
-    mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena);
-    if (arena_pages != NULL) {
-      size_t slice_index;
-      mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin];
-
-      if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) {
-        // found an abandoned page of the right size
-        // and claimed ownership.
-        mi_page_t* page = mi_arena_page_at_slice(arena, slice_index);
-        mi_assert_internal(mi_page_is_owned(page));
-        mi_assert_internal(mi_page_is_abandoned(page));
-        mi_assert_internal(mi_heap_has_page(heap, arena, page));
-        mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);
-        mi_theap_stat_decrease(theap, pages_abandoned, 1);
-        mi_theap_stat_counter_increase(theap, pages_reclaim_on_alloc, 1);
-
-        _mi_page_free_collect(page, false);  // update `used` count
-        mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
-        mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
-        mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
-        mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-        mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-        mi_assert_internal(mi_page_block_size(page) == block_size);
-        mi_assert_internal(!mi_page_is_full(page));
-        return page;
-      }
-    }
-  }
-  mi_forall_suitable_arenas_end();
-  return NULL;
-}
-
-static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid) {
-  MI_UNUSED_RELEASE(block_size);
-  const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena page
-  const size_t page_alignment = MI_ARENA_SLICE_ALIGN;
-
-  mi_heap_t*  const heap = _mi_theap_heap(theap);
-  mi_tld_t*   const tld  = theap->tld;
-  mi_arena_t* const req_arena = heap->exclusive_arena;
-  const int numa_node = (heap->numa_node >= 0 ? heap->numa_node : tld->numa_node);
-
-  // try to allocate from free space in arena's
-  uint8_t* start = NULL;
-  *memid = _mi_memid_none();
-  const size_t alloc_size = mi_size_of_slices(slice_count);
-  if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) &&       // allowed to allocate from arena's?
-      !os_align &&                                                   // not large alignment
-      slice_count <= mi_arena_max_object_size()/MI_ARENA_SLICE_SIZE) // and not too large
-  {
-    start = (uint8_t*)mi_arenas_try_alloc(heap, slice_count, page_alignment, commit, allow_large, req_arena, tld->thread_seq, numa_node, memid);
-    if (start != NULL) {
-      mi_arena_pages_t* const arena_pages = mi_heap_ensure_arena_pages(heap, memid->mem.arena.arena);
-      if (arena_pages==NULL) {
-        _mi_arenas_free(start, mi_size_of_slices(slice_count), *memid); // roll back
-        start = NULL;
-      }
-      else {
-        // note: the following assert should hold if we could check it atomically, but in a concurrent setting we may already allocate in slice_count
-        // mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages, memid->mem.arena.slice_index, memid->mem.arena.slice_count));
-        mi_assert_internal(mi_bitmap_is_clear(arena_pages->pages, memid->mem.arena.slice_index));
-        mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index);
-      }
-    }
-  }
-
-  // otherwise fall back to the OS
-  if (start == NULL) {
-    if (os_align) {
-      // note: slice_count already includes the page
-      mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment));
-      start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid);
-    }
-    else {
-      start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid);
-    }
-  }
-
-  if (start == NULL) return NULL;
-  mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN));
-  mi_assert_internal(!os_align || _mi_is_aligned(start + page_alignment, block_alignment));
-  return start;
-}
-
-static size_t mi_page_block_start(size_t block_size, bool os_align)
-{
-  #if MI_GUARDED
-  // in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size
-  // this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages)
-  const size_t os_page_size = _mi_os_page_size();
-  mi_assert_internal(MI_PAGE_ALIGN >= os_page_size);
-  if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) {
-    return _mi_align_up(mi_page_info_size(), os_page_size);
-  }
-  else
-  #endif
-  if (os_align) {
-    return MI_PAGE_ALIGN;
-  }
-  else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {
-    // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB)
-    return _mi_align_up(mi_page_info_size(), block_size);
-  }
-  else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) {
-    // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB)
-    return _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2);
-  }
-  else {
-    // otherwise start after the info
-    return mi_page_info_size();
-  }
-}
-
-// Allocate a fresh page
-static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit)
-{
-  const bool os_align        = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
-  const size_t alloc_size    = mi_size_of_slices(slice_count);
-  mi_memid_t memid           = _mi_memid_none();
-  uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,block_size,block_alignment,os_align,commit,&memid);
-  if (!slice_start) return NULL;
-
-  // guard page at the end of mimalloc page?
-  #if MI_SECURE>=5
-  mi_assert(alloc_size > _mi_os_secure_guard_page_size());
-  const size_t page_noguard_size = alloc_size - _mi_os_secure_guard_page_size();
-  if (memid.initially_committed) {
-    _mi_os_secure_guard_page_set_at(slice_start + page_noguard_size, memid);
-  }
-  #else
-  const size_t page_noguard_size = alloc_size;
-  #endif
-
-  // allocate the page meta info
-  mi_page_t* page = NULL;
-  bool page_meta_is_separate = false;
-  size_t block_start = 0;
-
-  // allocate page meta info at the arena start?
-  if (memid.memkind == MI_MEM_ARENA) {
-    mi_arena_t* const arena = memid.mem.arena.arena;
-    if (arena->pages_meta != NULL) {
-      mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0);
-      mi_page_t* const page_meta = &arena->pages_meta[memid.mem.arena.slice_index];
-      mi_assert_internal(page_meta->block_size == 0);
-      #if MI_PAGE_META_ALIGNED_FREE_SMALL
-      // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice,
-      // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`)
-      if (block_size > MI_SMALL_SIZE_MAX)
-      #endif
-      {
-        page = page_meta;
-        page_meta_is_separate = true;
-        block_start = 0;
-        #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET)
-        #define MI_PAGE_BLOCK_START_MAX_OFFSET  (8*MI_INTPTR_BITS) /* 512 */
-        #endif
-        if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) {
-          block_start += block_size;
-        }
-        mi_assert_internal(page->block_size == 0);
-        _mi_memzero_aligned(page, sizeof(*page));
-      }
-    }
-  }
-  if (page == NULL) {
-    // put page meta info in front of the slice
-    page = (mi_page_t*)slice_start;
-    block_start = mi_page_block_start(block_size, os_align);
-  }
-
-  // commit first block?
-  size_t commit_size = 0;
-  if (!memid.initially_committed) {
-    commit_size = _mi_align_up(block_start + block_size, MI_PAGE_MIN_COMMIT_SIZE);
-    if (commit_size > page_noguard_size) { commit_size = page_noguard_size; }
-    bool is_zero = false;
-    if mi_unlikely(!mi_arena_commit( mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) {
-      _mi_arenas_free(slice_start, alloc_size, memid);
-      return NULL;
-    }
-  }
-  if (!memid.initially_zero && !page_meta_is_separate) {
-    _mi_memzero_aligned(page, sizeof(*page));
-  }
-
-  // claimed free slices: initialize the page partly
-  if (!memid.initially_zero && memid.initially_committed) {
-    mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);
-  }
-  else if (memid.initially_committed) {
-    mi_track_mem_defined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);
-  }
-  #if MI_DEBUG > 1
-  if (memid.initially_zero && memid.initially_committed) {
-    if (!mi_mem_is_zero(slice_start, page_noguard_size)) {
-      _mi_error_message(EFAULT, "internal error: page memory was not zero initialized.\n");
-      memid.initially_zero = false;
-      if (block_start > 0) { _mi_memzero_aligned(page, sizeof(*page)); }
-    }
-  }
-  #endif
-  const size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size);
-  mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX);
-
-  // initialize
-  page->reserved = (uint16_t)reserved;
-  page->page_start = slice_start + block_start;
-  page->block_size = block_size;
-  page->slice_committed = commit_size;
-  page->memid = memid;
-  page->free_is_zero = memid.initially_zero;
-  mi_assert_internal(page->free==NULL);
-  mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page));
-  mi_assert_internal(mi_page_slice_start(page) == slice_start);
-
-  // and own it
-  mi_page_claim_ownership(page);
-
-  // register in the page map
-  if mi_unlikely(!_mi_page_map_register(page)) {
-    _mi_arenas_free( slice_start, alloc_size, memid );
-    return NULL;
-  }
-
-  // stats
-  mi_theap_stat_increase(theap, pages, 1);
-  mi_theap_stat_increase(theap, page_bins[_mi_page_stats_bin(page)], 1);
-
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-  mi_assert_internal(mi_page_block_size(page) == block_size);
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(mi_page_is_owned(page));
-
-  return page;
-}
-
-// Allocate a regular small/medium/large page.
-static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_count, size_t block_size)
-{
-  // 1. look for an abandoned page
-  mi_page_t* page = mi_arenas_page_try_find_abandoned(theap, slice_count, block_size);
-  if (page != NULL) {
-    return page;  // return as abandoned
-  }
-
-  // 2. find a free block, potentially allocating a new arena
-  const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand);
-  const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) ||  // always commit small pages
-                       (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0));
-  page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit);
-  if (page == NULL) return NULL;
-
-  mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count);
-  if (!_mi_page_init(theap, page)) {
-    _mi_arenas_free( page, mi_page_full_size(page), page->memid);
-    return NULL;
-  }
-
-  return page;
-}
-
-// Allocate a page containing one block (very large, or with large alignment)
-static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment)
-{
-  const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
-  const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size());
-  #if MI_SECURE < 2
-  const size_t slice_count = mi_slice_count_of_size(info_size + block_size);
-  #else
-  const size_t slice_count = mi_slice_count_of_size(_mi_align_up(info_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size());
-  #endif
-
-  mi_page_t* page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, block_alignment, true /* commit singletons always */);
-  if (page == NULL) return NULL;
-
-  mi_assert(page->reserved == 1);
-  if (!_mi_page_init(theap, page)) {
-    _mi_arenas_free( page, mi_page_full_size(page), page->memid);
-    return NULL;
-  }
-
-  return page;
-}
-
-
-mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) {
-  mi_page_t* page;
-  if mi_unlikely(block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
-    mi_assert_internal(_mi_is_power_of_two(block_alignment));
-    page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);
-  }
-  else if (block_size <= MI_SMALL_MAX_OBJ_SIZE) {
-    page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_SMALL_PAGE_SIZE), block_size);
-  }
-  else if (block_size <= MI_MEDIUM_MAX_OBJ_SIZE) {
-    page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_MEDIUM_PAGE_SIZE), block_size);
-  }
-  #if MI_ENABLE_LARGE_PAGES
-  else if (block_size <= MI_LARGE_MAX_OBJ_SIZE) {
-    page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_LARGE_PAGE_SIZE), block_size);
-  }
-  #endif
-  else {
-    page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);
-  }
-  if mi_unlikely(page == NULL) {
-    return NULL;
-  }
-  // mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc == tld->subproc);
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-  mi_assert_internal(block_alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN || _mi_is_aligned(mi_page_start(page), block_alignment));
-
-  return page;
-}
-
-void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(mi_page_all_free(page));
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(page->next==NULL && page->prev==NULL);
-  mi_assert_internal(current_theapx == NULL || _mi_thread_id()==current_theapx->tld->thread_id);
-
-  if (current_theapx != NULL) {
-    mi_theap_stat_decrease(current_theapx, page_bins[_mi_page_stats_bin(page)], 1);
-    mi_theap_stat_decrease(current_theapx, pages, 1);
-  }
-  else {
-    mi_heap_t* const heap = mi_page_heap(page);
-    mi_heap_stat_decrease(heap, page_bins[_mi_page_stats_bin(page)], 1);
-    mi_heap_stat_decrease(heap, pages, 1);
-  }
-
-  #if MI_DEBUG>1
-  if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
-    size_t bin = _mi_bin(mi_page_block_size(page));
-    size_t slice_index;
-    size_t slice_count;
-    mi_arena_pages_t* arena_pages = NULL;
-    mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
-    mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
-    mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
-    mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));
-    mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1));
-    // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may
-    // be (temporarily) not true if the free happens while trying to reclaim
-    // see `mi_arena_try_claim_abandoned`
-  }
-  #endif
-
-  // recommit guard page at the end?
-  // we must do this since we may later allocate large spans over this page and cannot have a guard page in between
-  #if MI_SECURE >= 5
-  if (!page->memid.is_pinned) {
-    _mi_os_secure_guard_page_reset_before(mi_page_slice_start(page) + mi_page_full_size(page), page->memid);
-  }
-  #endif
-
-  // unregister page
-  _mi_page_map_unregister(page);
-  if (page->memid.memkind == MI_MEM_ARENA) {
-    mi_arena_pages_t* arena_pages;
-    size_t slice_index;
-    size_t slice_count; MI_UNUSED(slice_count);
-    mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
-    mi_assert_internal(arena_pages!=NULL);
-    mi_bitmap_clear(arena_pages->pages, slice_index);
-    if (page->slice_committed > 0) {
-      // if committed on-demand, set the commit bits to account commit properly
-      mi_assert_internal(mi_page_full_size(page) >= page->slice_committed);
-      const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE;  // conservative
-      //mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices));
-      mi_assert_internal(slice_count >= total_slices);
-      if (total_slices > 0) {
-        mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL);
-      }
-      // any left over?
-      const size_t extra = page->slice_committed % MI_ARENA_SLICE_SIZE;
-      if (extra > 0) {
-        // pretend it was decommitted already
-        mi_subproc_stat_decrease(arena->subproc, committed, extra);
-      }
-    }
-    else {
-      mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
-    }
-  }
-  if (mi_page_meta_is_separated(page)) { page->block_size = 0; }  // for assertion checking
-  _mi_arenas_free( mi_page_slice_start(page), mi_page_full_size(page), page->memid);
-}
-
-/* -----------------------------------------------------------
-  Arena abandon
------------------------------------------------------------ */
-
-void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) {
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(!mi_page_all_free(page));
-  mi_assert_internal(page->next==NULL && page->prev == NULL);
-  mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);
-  // mi_assert_internal(current_theap == _mi_page_associated_theap(page));
-
-  mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(heap==_mi_theap_heap(current_theap));
-  if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
-    // make available for allocations
-    size_t bin = _mi_bin(mi_page_block_size(page));
-    size_t slice_index;
-    size_t slice_count;
-    mi_arena_pages_t* arena_pages = NULL;
-    mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);
-
-    mi_assert_internal(!mi_page_is_singleton(page));
-    mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
-    mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
-    mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
-
-    mi_page_set_abandoned_mapped(page);
-    const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index);
-    MI_UNUSED(was_clear); mi_assert_internal(was_clear);
-    mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);
-    mi_theap_stat_increase(current_theap, pages_abandoned, 1);
-  }
-  else {
-    // page is full (or a singleton), or the page is OS/externally allocated
-    // leave as is; it will be reclaimed when an object is free'd in the page
-    // but for non-arena pages, add to the subproc list so these can be visited
-    if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {
-      mi_lock(&heap->os_abandoned_pages_lock) {
-        // push in front
-        page->prev = NULL;
-        page->next = heap->os_abandoned_pages;
-        if (page->next != NULL) { page->next->prev = page; }
-        heap->os_abandoned_pages = page;
-      }
-    }
-    mi_theap_stat_increase(current_theap, pages_abandoned, 1);
-  }
-  mi_abandoned_page_unown(page, current_theap);
-}
-
-
-// this is called from `free.c:mi_free_try_collect_mt` only.
-bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) {
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(!mi_page_is_abandoned_mapped(page));
-  mi_assert_internal(!mi_page_is_full(page));
-  mi_assert_internal(!mi_page_all_free(page));
-  mi_assert_internal(!mi_page_is_singleton(page));
-  if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) {
-    return false;
-  }
-  else {
-    // do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap
-    mi_theap_t* const theap = _mi_page_associated_theap_peek(page);
-    if (theap == NULL) {
-      return false;
-    }
-    else {
-      mi_theap_stat_counter_increase(theap, pages_reabandon_full, 1);
-      mi_theap_stat_adjust_decrease(theap, pages_abandoned, 1);  // adjust as we are not abandoning fresh
-      _mi_arenas_page_abandon(page, theap);
-      return true;
-    }
-  }
-}
-
-// called from `mi_free` if trying to unabandon an abandoned page
-void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(current_theapx==NULL || _mi_thread_id()==current_theapx->tld->thread_id);
-
-  mi_heap_t* const heap = mi_page_heap(page);
-  if (mi_page_is_abandoned_mapped(page)) {
-    mi_assert_internal(page->memid.memkind==MI_MEM_ARENA);
-    // remove from the abandoned map
-    size_t bin = _mi_bin(mi_page_block_size(page));
-    size_t slice_index;
-    size_t slice_count;
-    mi_arena_pages_t* arena_pages;
-    mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);  MI_UNUSED(arena);
-
-    mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
-    mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
-
-    // this busy waits until a concurrent reader (from alloc_abandoned) is done
-    mi_bitmap_clear_once_set(arena_pages->pages_abandoned[bin], slice_index);
-    mi_page_clear_abandoned_mapped(page);
-    mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);
-  }
-  else {
-    // page is full (or a singleton), page is OS allocated
-    // if not an arena page, remove from the subproc os pages list
-    if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {
-      mi_lock(&heap->os_abandoned_pages_lock) {
-        if (page->prev != NULL) { page->prev->next = page->next; }
-        if (page->next != NULL) { page->next->prev = page->prev; }
-        if (heap->os_abandoned_pages == page) { heap->os_abandoned_pages = page->next; }
-        page->next = NULL;
-        page->prev = NULL;
-      }
-    }
-  }
-  if (current_theapx!=NULL) {
-    mi_theap_stat_decrease(current_theapx, pages_abandoned, 1);
-  }
-  else {
-    mi_heap_stat_decrease(heap, pages_abandoned, 1);
-  }
-}
-
-
-/* -----------------------------------------------------------
-  Arena free
------------------------------------------------------------ */
-static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices);
-static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq);
-
-void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {
-  if (p==NULL) return;
-  if (size==0) return;
-
-  // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.)
-  mi_track_mem_undefined(p, size);
-
-  if (mi_memkind_is_os(memid.memkind)) {
-    // was a direct OS allocation, pass through
-    _mi_os_free(p, size, memid);
-  }
-  else if (memid.memkind == MI_MEM_ARENA) {
-    // allocated in an arena
-    size_t slice_count;
-    size_t slice_index;
-    mi_arena_t* arena = mi_arena_from_memid(memid, &slice_index, &slice_count);
-    mi_assert_internal((size%MI_ARENA_SLICE_SIZE)==0);
-    mi_assert_internal((slice_count*MI_ARENA_SLICE_SIZE)==size);
-    mi_assert_internal(mi_arena_slice_start(arena,slice_index) <= (uint8_t*)p);
-    mi_assert_internal(mi_arena_slice_start(arena,slice_index) + mi_size_of_slices(slice_count) > (uint8_t*)p);
-    // checks
-    if (arena == NULL) {
-      _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memid: 0x%zx\n", p, size, memid);
-      return;
-    }
-    mi_assert_internal(slice_index < arena->slice_count);
-    mi_assert_internal(slice_index >= mi_arena_info_slices(arena));
-    if (slice_index < mi_arena_info_slices(arena) || slice_index >= arena->slice_count) {
-      _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid);
-      return;
-    }
-
-    // potentially decommit
-    if (!arena->memid.is_pinned /* && !arena->memid.initially_committed */) { // todo: allow decommit even if initially committed?
-      // (delay) purge the page
-      mi_arena_schedule_purge(arena, slice_index, slice_count);
-    }
-
-    // and make it available to others again
-    bool all_inuse = mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);
-    if (!all_inuse) {
-      _mi_error_message(EAGAIN, "trying to free an already freed arena block: %p, size %zu\n", mi_arena_slice_start(arena,slice_index), mi_size_of_slices(slice_count));
-      return;
-    };
-  }
-  else if (memid.memkind == MI_MEM_META) {
-    _mi_meta_free(p, size, memid);
-  }
-  else {
-    // arena was none, external, or static; nothing to do
-    mi_assert_internal(mi_memid_needs_no_free(memid));
-  }
-
-  // try to purge expired decommits
-  // mi_arenas_try_purge(false, false, NULL);
-}
-
-// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired
-void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld) {
-  mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq);
-}
-
-
-// Is a pointer contained in the given arena area?
-static bool mi_arena_strictly_contains(mi_arena_t* arena, const void* p) {
-  return (arena != NULL &&
-          mi_arena_start(arena) <= (const uint8_t*)p &&
-          mi_arena_start(arena) + mi_size_of_slices(arena->slice_count) >(const uint8_t*)p);
-}
-
-// Is a pointer inside any of our arenas?
-static bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) {
-  mi_subproc_t* subproc = _mi_subproc();
-  const size_t max_arena = mi_arenas_get_count(subproc);
-  for (size_t i = 0; i < max_arena; i++) {
-    mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
-    if (arena != NULL) {
-      if (parent==NULL || arena==parent || arena->parent==parent) {
-        if (mi_arena_strictly_contains(arena, p)) {
-          return true;
-        }
-      }
-    }
-  }
-  return false;
-}
-
-// Is a pointer inside any of our arenas?
-bool _mi_arenas_contain(const void* p) {
-  return mi_arenas_contain_ex(p, NULL);
-}
-
-// Is a pointer contained in the given arena area?
-bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) {
-  mi_arena_t* arena = _mi_arena_from_id(arena_id);
-  if (arena==NULL) return false;
-  else if (mi_arena_strictly_contains(arena, p)) return true;
-  else return mi_arenas_contain_ex(p, arena);  // maybe a subarena?
-}
-
-
-/* -----------------------------------------------------------
-  Remove an arena.
------------------------------------------------------------ */
-
-// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`
-// for dynamic libraries that are unloaded and need to release all their allocated memory.
-static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) {
-  mi_assert_internal(subproc != NULL);
-  const size_t arena_count = mi_arenas_get_count(subproc);
-  for (size_t i = 0; i < arena_count; i++) {
-    mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
-    if (arena != NULL) {
-      // mi_lock_done(&arena->abandoned_visit_lock);
-      mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);
-      if (mi_memkind_is_os(arena->memid.memkind)) {
-        _mi_os_free_ex(mi_arena_start(arena), mi_arena_size(arena), true, arena->memid, subproc); // pass `subproc` to avoid accessing the theap pointer (in `_mi_subproc()`)
-      }
-    }
-  }
-  // try to lower the max arena.
-  size_t expected = arena_count;
-  mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, (size_t)0);
-}
-
-
-// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`
-// for dynamic libraries that are unloaded and need to release all their allocated memory.
-void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc) {
-  mi_arenas_unsafe_destroy(subproc);
-  // mi_arenas_try_purge(true /* force purge */, true /* visit all*/, subproc, 0 /* thread seq */);  // purge non-owned arenas
-}
-
-
-/* -----------------------------------------------------------
-  Add an arena.
------------------------------------------------------------ */
-
-static bool mi_arenas_add(mi_subproc_t* subproc, mi_arena_t* arena, mi_arena_id_t* arena_id)
-{
-  mi_assert_internal(arena != NULL);
-  mi_assert_internal(arena->slice_count > 0);
-  if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }
-
-  // try to find a NULL entry
-  mi_arena_t* expected;
-  size_t count = mi_arenas_get_count(subproc);
-  for( size_t i = 0; i < count; i++) {
-    if (mi_arena_from_index(subproc,i) == NULL) {
-      arena->arena_idx = i;
-      expected = NULL;
-      if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[i], &expected, arena)) {
-        // success
-        if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); }
-        return true;
-      }
-    }
-  }
-
-  // otherwise, try to allocate a fresh slot
-  while(count<MI_MAX_ARENAS) {
-    if (mi_atomic_cas_strong_release(&subproc->arena_count, &count, count+1)) {
-      arena->arena_idx = count;
-      expected = NULL;
-      if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[count], &expected, arena)) {
-        mi_subproc_stat_counter_increase(arena->subproc, arena_count, 1);
-        if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); }
-        return true;
-      }
-    }
-  }
-
-  // failed
-  arena->arena_idx = 0;
-  arena->subproc = NULL;
-  return false;
-}
-
-static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) {
-  if (slice_count == 0) slice_count = MI_BCHUNK_BITS;
-  mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);
-  const size_t base_size = _mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE);
-  const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned
-  const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL);
-  const size_t size = base_size + bitmaps_size;
-  if (bitmap_base != NULL) *bitmap_base = base_size;
-  return size;
-}
-
-static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) {
-  if (slice_count == 0) slice_count = MI_BCHUNK_BITS;
-  mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);
-  const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE);
-  const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned
-  const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free
-  #if MI_PAGE_META_IS_SEPARATED
-  const size_t pages_size = slice_count * sizeof(mi_page_t);
-  #else
-  const size_t pages_size = 0;
-  #endif
-  const size_t size = base_size + bitmaps_size + pages_size;
-
-  const size_t os_page_size = _mi_os_page_size();
-  const size_t info_size = _mi_align_up(size, os_page_size) + _mi_os_secure_guard_page_size();
-  const size_t info_slices = mi_slice_count_of_size(info_size);
-
-  if (bitmap_base != NULL) *bitmap_base = base_size;
-  return info_slices;
-}
-
-static mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) {
-  mi_bitmap_t* bitmap = (mi_bitmap_t*)(*base);
-  *base = (*base) + mi_bitmap_init(bitmap, slice_count, true /* already zero */);
-  return bitmap;
-}
-
-static mi_bbitmap_t* mi_arena_bbitmap_init(size_t slice_count, uint8_t** base) {
-  mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base);
-  *base = (*base) + mi_bbitmap_init(bbitmap, slice_count, true /* already zero */);
-  return bbitmap;
-}
-
-static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) {
-  const size_t slice_count = arena->slice_count;
-  size_t bitmap_base = 0;
-  const size_t size = mi_arena_pages_size(slice_count, &bitmap_base);
-  mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(mi_heap_main(), size, MI_BCHUNK_SIZE);
-  if (arena_pages==NULL) return NULL;
-  uint8_t* base = (uint8_t*)arena_pages + bitmap_base;
-  mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE));
-  arena_pages->pages = mi_arena_bitmap_init(slice_count, &base);
-  for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
-    arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);
-  }
-  return arena_pages;
-}
-
-static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
-                                        size_t slice_count, mi_arena_t* parent, size_t total_size,
-                                        int numa_node, bool exclusive,
-                                        mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id)
-{
-  mi_assert_internal(_mi_is_aligned(start,MI_ARENA_SLICE_ALIGN));
-  mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE);
-
-  if (slice_count > MI_BITMAP_MAX_BIT_COUNT) {  // 16 GiB for now
-    // note: this should never happen if called from `mi_manage_os_memory` (as that allocates sub-arenas when needed)
-    _mi_warning_message("cannot use OS memory since it is too large (size %zu MiB, maximum is %zu MiB)", mi_size_of_slices(slice_count)/MI_MiB, mi_size_of_slices(MI_BITMAP_MAX_BIT_COUNT)/MI_MiB);
-    return NULL;
-  }
-
-  size_t bitmap_base;
-  const size_t info_slices = mi_arena_info_slices_needed(slice_count, &bitmap_base);
-  if (slice_count < info_slices+1) {
-    _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", mi_size_of_slices(slice_count)/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB);
-    return NULL;
-  }
-  else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) {
-    _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES);
-    return NULL;
-  }
-
-  mi_arena_t* arena = (mi_arena_t*)start;
-
-  // commit & zero if needed
-  if (!memid.initially_committed) {
-    size_t commit_size = mi_size_of_slices(info_slices);
-    // leave a guard OS page decommitted at the end?
-    if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); }
-    bool ok = false;
-    if (commit_fun != NULL) {
-      ok = (*commit_fun)(true /* commit */, arena, commit_size, NULL, commit_fun_arg);
-    }
-    else {
-      ok = _mi_os_commit(arena, commit_size, NULL);
-    }
-    if (!ok) {
-      _mi_warning_message("unable to commit meta-data for OS memory");
-      return NULL;
-    }
-  }
-  else if (!memid.is_pinned) {
-    // if MI_SECURE, set a guard page at the end of the arena info
-    // todo: this does not respect the commit_fun as the memid is of external memory
-    _mi_os_secure_guard_page_set_before((uint8_t*)arena + mi_size_of_slices(info_slices), memid);
-  }
-  if (!memid.initially_zero) {
-    _mi_memzero(arena, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size());
-  }
-
-  // init
-  arena->subproc = subproc;
-  arena->memid = memid;
-  arena->is_exclusive = exclusive;
-  arena->slice_count = slice_count;
-  arena->info_slices = info_slices;
-  if (numa_node<0 && mi_option_is_enabled(mi_option_arena_is_numa_local)) {
-    arena->numa_node = _mi_os_numa_node();
-  }
-  else {
-    arena->numa_node = numa_node;
-  }
-  arena->purge_expire = 0;
-  arena->commit_fun = commit_fun;
-  arena->commit_fun_arg = commit_fun_arg;
-  arena->parent = parent;
-  arena->total_size = total_size;
-
-  // init bitmaps
-  uint8_t* base = mi_arena_start(arena) + bitmap_base;
-  arena->slices_free = mi_arena_bbitmap_init(slice_count, &base);
-  arena->slices_committed = mi_arena_bitmap_init(slice_count, &base);
-  arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base);
-  arena->slices_purge = mi_arena_bitmap_init(slice_count, &base);
-  arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base);
-  for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
-    arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);
-  }
-  #if MI_PAGE_META_IS_SEPARATED
-  arena->pages_meta = (mi_page_t*)base;
-  base += (slice_count * sizeof(mi_page_t));
-  #else
-  arena->pages_meta = NULL;
-  #endif
-  mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena)));
-
-  // reserve our meta info (and reserve slices outside the memory area)
-  mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices);
-  if (memid.initially_committed) {
-    mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count);
-  }
-  if (!memid.initially_zero) {
-    mi_bitmap_unsafe_setN(arena->slices_dirty, 0, arena->slice_count);
-  }
-
-  if (!mi_arenas_add(subproc, arena, arena_id)) { return NULL;  }
-  return arena;
-}
-
-static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t size, int numa_node, bool exclusive,
-  mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept
-{
-  // checks
-  mi_assert(_mi_is_aligned(start, MI_ARENA_SLICE_SIZE));
-  mi_assert(start!=NULL);
-  if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }
-  if (start==NULL) return false;
-  if (!_mi_is_aligned(start, MI_ARENA_SLICE_SIZE)) {
-    // we can align the start since the memid tracks the real base of the memory.
-    void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_SLICE_SIZE);
-    const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start;
-    if (diff >= size || (size - diff) < MI_ARENA_SLICE_SIZE) {
-      _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size);
-      return false;
-    }
-    start = aligned_start;
-    size = size - diff;
-  }
-
-  // allocate enough arena's to span the full memory area
-  // the first arena is the owner, the rest are "sub-arena" (with `parent` pointing to the first one)
-  size_t total_slice_count = _mi_align_down(size / MI_ARENA_SLICE_SIZE, MI_BCHUNK_BITS);
-  size_t total_size = mi_size_of_slices(total_slice_count);
-  if (total_size < MI_ARENA_MIN_SIZE) {
-    _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", size/MI_KiB, MI_ARENA_MIN_SIZE/MI_KiB);
-    return false;
-  }
-
-  mi_arena_t* parent = NULL;
-  do {
-    // counting down on the total_slice_count
-    size_t slice_count = total_slice_count;
-    if (slice_count > MI_BITMAP_MAX_BIT_COUNT) {  // 16 GiB for now (with 64KiB slices)
-      slice_count = MI_BITMAP_MAX_BIT_COUNT;
-    }
-
-    // initialize
-    mi_arena_t* arena = mi_arena_initialize( subproc, start, slice_count, parent,
-                                              (parent==NULL ? total_size : 0), numa_node, exclusive,
-                                              memid, commit_fun, commit_fun_arg,
-                                              (parent==NULL ? arena_id : NULL));
-    if (arena==NULL) {
-      // failed to initialize due to failing commit or too many arena's
-      if (parent==NULL) {
-        return false;
-      }
-      else {
-        // partial success, but failed to use the full area..
-        // todo: roll-back in this case? that requires a lock on the arena's array though
-        mi_assert(mi_size_of_slices(total_slice_count) <= parent->total_size);
-        parent->total_size -= mi_size_of_slices(total_slice_count);
-        return true;
-      }
-    }
-
-    // success
-    if (parent==NULL) {
-      parent = arena;
-      memid.memkind = MI_MEM_NONE;
-    }
-    mi_assert(slice_count <= total_slice_count);
-    total_slice_count -= slice_count;
-    start = (uint8_t*)start + mi_size_of_slices(slice_count);
-  }
-  while (total_slice_count > 0);
-
-  return true;
-}
-
-bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {
-  mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL);
-  memid.mem.os.base = start;
-  memid.mem.os.size = size;
-  memid.initially_committed = is_committed;
-  memid.initially_zero = is_zero;
-  memid.is_pinned = is_pinned;
-  return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, NULL, NULL, arena_id);
-}
-
-bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept
-{
-  mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL);
-  memid.mem.os.base = start;
-  memid.mem.os.size = size;
-  memid.initially_committed = is_committed;
-  memid.initially_zero = is_zero;
-  memid.is_pinned = is_pinned;
-  return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, commit_fun, commit_fun_arg, arena_id);
-}
-
-
-// Reserve a range of regular OS memory
-static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) {
-  if (arena_id != NULL) *arena_id = _mi_arena_id_none();
-  size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice
-  mi_memid_t memid;
-  void* start = _mi_os_alloc_aligned(size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid);
-  if (start == NULL) return ENOMEM;
-  if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) {
-    _mi_os_free_ex(start, size, commit, memid, NULL);
-    _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024));
-    return ENOMEM;
-  }
-  _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size, 1024), memid.is_pinned ? " (in large os pages)" : "");
-  // mi_debug_show_arenas(true, true, false);
-
-  return 0;
-}
-
-// Reserve a range of regular OS memory
-int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {
-  return mi_reserve_os_memory_ex2(_mi_subproc(), size, commit, allow_large, exclusive, arena_id);
-}
-
-// Manage a range of regular OS memory
-bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept {
-  return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL);
-}
-
-// Reserve a range of regular OS memory
-int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept {
-  return mi_reserve_os_memory_ex(size, commit, allow_large, false, NULL);
-}
-
-
-/* -----------------------------------------------------------
-  Debugging
------------------------------------------------------------ */
-
-// Return idx of the slice past the last used slice
-static size_t mi_arena_used_slices(mi_arena_t* arena) {
-  size_t idx;
-  if (mi_bbitmap_bsr_inv(arena->slices_free, &idx)) {
-    return (idx + 1);
-  }
-  else {
-    return mi_arena_info_slices(arena);
-  }
-}
-
-static size_t mi_debug_show_bfield(mi_bfield_t field, char* buf, size_t* k) {
-  size_t bit_set_count = 0;
-  for (int bit = 0; bit < MI_BFIELD_BITS; bit++) {
-    bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0);
-    if (is_set) bit_set_count++;
-    buf[*k] = (is_set ? 'x' : '.');
-    *k = *k + 1;
-  }
-  return bit_set_count;
-}
-
-typedef enum mi_ansi_color_e {
-  MI_BLACK = 30,
-  MI_MAROON,
-  MI_DARKGREEN,
-  MI_ORANGE,
-  MI_NAVY,
-  MI_PURPLE,
-  MI_TEAL,
-  MI_GRAY,
-  MI_DARKGRAY = 90,
-  MI_RED,
-  MI_GREEN,
-  MI_YELLOW,
-  MI_BLUE,
-  MI_MAGENTA,
-  MI_CYAN,
-  MI_WHITE
-} mi_ansi_color_t;
-
-static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) {
-  *k += _mi_snprintf(buf + *k, 32, "\x1B[%dm", (int)color);
-}
-
-static int mi_page_commit_usage(mi_page_t* page) {
-  // if (mi_page_size(page) <= MI_PAGE_MIN_COMMIT_SIZE) return 100;
-  const size_t committed_size = mi_page_committed(page);
-  const size_t used_size = page->used * mi_page_block_size(page);
-  return (int)(used_size * 100 / committed_size);
-}
-
-static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, size_t slice_index, long* pbit_of_page, mi_ansi_color_t* pcolor_of_page ) {
-  size_t bit_set_count = 0;
-  long bit_of_page = *pbit_of_page;
-  mi_ansi_color_t color = *pcolor_of_page;
-  mi_ansi_color_t prev_color = MI_GRAY;
-  for (int bit = 0; bit < MI_BFIELD_BITS; bit++, bit_of_page--) {
-    // bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0);
-    void* start = mi_arena_slice_start(arena, slice_index + bit);
-    mi_page_t* page = _mi_safe_ptr_page(start);
-    char c = ' ';
-    if (page!=NULL && start==mi_page_slice_start(page)) {
-      mi_assert_internal(bit_of_page <= 0);
-      bit_set_count++;
-      c = 'p';
-      color = MI_GRAY;
-      if (mi_page_is_singleton(page)) { c = 's'; }
-      else if (mi_page_is_full(page)) { c = 'f'; }
-      if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); }
-      int commit_usage = mi_page_commit_usage(page);
-      if (commit_usage < 25) { color = MI_MAROON; }
-      else if (commit_usage < 50) { color = MI_ORANGE; }
-      else if (commit_usage < 75) { color = MI_TEAL; }
-      else color = MI_DARKGREEN;
-      bit_of_page = (long)page->memid.mem.arena.slice_count;
-    }
-    else {
-      c = '?';
-      if (bit_of_page > 0) { c = '-'; }
-      else if (_mi_meta_is_meta_page(start)) { c = 'm'; color = MI_GRAY; }
-      else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; }
-      // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; }
-      else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) {
-        if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; }
-        else if (mi_bitmap_is_set(arena->slices_committed, slice_index + bit)) { c = '_'; color = MI_GRAY; }
-        else { c = '.'; color = MI_GRAY; }
-      }
-      if (bit==MI_BFIELD_BITS-1 && bit_of_page > 1) { c = '>'; }
-    }
-    if (color != prev_color) {
-      mi_debug_color(buf, k, color);
-      prev_color = color;
-    }
-    buf[*k] = c; *k += 1;
-  }
-  mi_debug_color(buf, k, MI_GRAY);
-  *pbit_of_page = bit_of_page;
-  *pcolor_of_page = color;
-  return bit_set_count;
-}
-
-static size_t mi_debug_show_chunks(const char* header1, const char* header2, const char* header3,
-                                   size_t slice_count, size_t chunk_count,
-                                   mi_bchunk_t* chunks, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow)
-{
-  _mi_raw_message("\x1B[37m%s%s%s (use/commit: \x1B[31m0 - 25%%\x1B[33m - 50%%\x1B[36m - 75%%\x1B[32m - 100%%\x1B[0m)\n", header1, header2, header3);
-  const size_t fields_per_line = (narrow ? 2 : 4);
-  const size_t used_slice_count = mi_arena_used_slices(arena);
-  size_t bit_count = 0;
-  size_t bit_set_count = 0;
-  long bit_of_page = 0;
-  mi_ansi_color_t color_of_page = MI_GRAY;
-  for (size_t i = 0; i < chunk_count && bit_count < slice_count; i++) {
-    char buf[5*MI_BCHUNK_BITS + 64]; _mi_memzero(buf, sizeof(buf));
-    if (bit_count > used_slice_count && i+2 < chunk_count) {
-      const size_t diff = chunk_count - 1 - i;
-      bit_count += diff*MI_BCHUNK_BITS;
-      _mi_raw_message("  |\n");
-      i = chunk_count-1;
-    }
-
-    size_t k = 0;
-
-    if (i<10)        { buf[k++] = ('0' + (char)i); buf[k++] = ' '; buf[k++] = ' '; }
-    else if (i<100)  { buf[k++] = ('0' + (char)(i/10)); buf[k++] = ('0' + (char)(i%10)); buf[k++] = ' '; }
-    else if (i<1000) { buf[k++] = ('0' + (char)(i/100)); buf[k++] = ('0' + (char)((i%100)/10)); buf[k++] = ('0' + (char)(i%10)); }
-
-    char chunk_kind = ' ';
-    if (chunk_bins != NULL) {
-      switch (mi_bbitmap_debug_get_bin(chunk_bins,i)) {
-        case MI_CBIN_SMALL:  chunk_kind = 'S'; break;
-        case MI_CBIN_MEDIUM: chunk_kind = 'M'; break;
-        case MI_CBIN_LARGE:  chunk_kind = 'L'; break;
-        case MI_CBIN_HUGE:   chunk_kind = 'H'; break;
-        case MI_CBIN_OTHER:  chunk_kind = 'X'; break;
-        default: chunk_kind = ' '; break; // suppress warning
-        // case MI_CBIN_NONE: chunk_kind = 'N'; break;
-      }
-    }
-    buf[k++] = chunk_kind;
-    buf[k++] = ' ';
-
-    for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
-      if (j > 0 && (j % fields_per_line) == 0) {
-        // buf[k++] = '\n'; _mi_memset(buf+k,' ',7); k += 7;
-        _mi_raw_message("  %s\n\x1B[37m", buf);
-        _mi_memzero(buf, sizeof(buf));
-        _mi_memset(buf, ' ', 5); k = 5;
-      }
-      if (bit_count < slice_count) {
-        mi_bfield_t bfield = 0;
-        if (chunks!=NULL) {
-          bfield = chunks[i].bfields[j];
-        }
-        if (invert) bfield = ~bfield;
-        size_t xcount = (chunks==NULL ? mi_debug_show_page_bfield(buf, &k, arena, bit_count, &bit_of_page, &color_of_page)
-                                      : mi_debug_show_bfield(bfield, buf, &k));
-        if (invert) xcount = MI_BFIELD_BITS - xcount;
-        bit_set_count += xcount;
-        buf[k++] = ' ';
-      }
-      else {
-        _mi_memset(buf + k, 'o', MI_BFIELD_BITS);
-        k += MI_BFIELD_BITS;
-      }
-      bit_count += MI_BFIELD_BITS;
-    }
-    _mi_raw_message("  %s\n\x1B[37m", buf);
-  }
-  _mi_raw_message("\x1B[0m  total pages: %zu\n", bit_set_count);
-  return bit_set_count;
-}
-
-//static size_t mi_debug_show_bitmap_binned(const char* header1, const char* header2, const char* header3, size_t slice_count,
-//                                           mi_bitmap_t* bitmap, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow) {
-//  return mi_debug_show_chunks(header1, header2, header3, slice_count, mi_bitmap_chunk_count(bitmap), &bitmap->chunks[0], chunk_bins, invert, arena, narrow);
-//}
-
-static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narrow) mi_attr_noexcept {
-  mi_subproc_t* subproc = heap->subproc;
-  size_t max_arenas = mi_arenas_get_count(subproc);
-  //size_t free_total = 0;
-  //size_t slice_total = 0;
-  //size_t abandoned_total = 0;
-  size_t page_total = 0;
-  for (size_t i = 0; i < max_arenas; i++) {
-    mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
-    if (arena == NULL) break;
-    mi_assert(arena->subproc == subproc);
-    // slice_total += arena->slice_count;
-    _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %p, numa: %i\n",
-        (arena->parent==NULL ? "" : "(sub)"), i, arena, arena->slice_count, (size_t)(mi_size_of_slices(arena->slice_count)/MI_MiB),
-        (arena->memid.is_pinned ? ", pinned" : ""), (arena->is_exclusive ? ", exclusive" : ""),
-        arena->subproc, arena->numa_node);
-    //if (show_inuse) {
-    //  free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL);
-    //}
-    //if (show_committed) {
-    //  mi_debug_show_bitmap("committed slices", arena->slice_count, arena->slices_committed, false, NULL);
-    //}
-    // todo: abandoned slices
-    //if (show_purge) {
-    //  purge_total += mi_debug_show_bitmap("purgeable slices", arena->slice_count, arena->slices_purge, false, NULL);
-    //}
-    if (show_pages) {
-      // mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);
-      // if (arena_pages != NULL)
-      {
-        const char* header1 = "chunks (p:page, f:full, s:singleton, P,F,S:not abandoned, i:arena-info, m:meta-data, ~:free-purgable, _:free-committed, .:free-reserved)";
-        const char* header2 = (narrow ? "\n       " : " ");
-        const char* header3 = "(chunk bin: S:small, M : medium, L : large, X : other)";
-        page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count,
-                                           mi_bbitmap_chunk_count(arena->slices_free), NULL,
-                                           arena->slices_free->chunkmap_bins, false, arena, narrow);
-      }
-    }
-  }
-  // if (show_inuse)     _mi_raw_message("total inuse slices    : %zu\n", slice_total - free_total);
-  // if (show_abandoned) _mi_raw_message("total abandoned slices: %zu\n", abandoned_total);
-  if (show_pages) _mi_raw_message("total pages in arenas: %zu\n", page_total);
-}
-
-void mi_debug_show_arenas(void) mi_attr_noexcept {
-  mi_debug_show_arenas_ex(mi_heap_main(), true /* show pages */, true /* narrow? */);
-}
-
-void mi_arenas_print(void) mi_attr_noexcept {
-  mi_debug_show_arenas();
-}
-
-
-/* -----------------------------------------------------------
-  Reserve a huge page arena.
------------------------------------------------------------ */
-// reserve at a specific numa node
-int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {
-  if (arena_id != NULL) *arena_id = NULL;
-  if (pages==0) return 0;
-  if (numa_node < -1) numa_node = -1;
-  if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count();
-  size_t hsize = 0;
-  size_t pages_reserved = 0;
-  mi_memid_t memid;
-  void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid);
-  if (p==NULL || pages_reserved==0) {
-    _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages);
-    return ENOMEM;
-  }
-  _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages);
-
-  if (!mi_manage_os_memory_ex2(_mi_subproc(), p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) {
-    _mi_os_free(p, hsize, memid);
-    return ENOMEM;
-  }
-  return 0;
-}
-
-int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept {
-  return mi_reserve_huge_os_pages_at_ex(pages, numa_node, timeout_msecs, false, NULL);
-}
-
-// reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)
-int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept {
-  if (pages == 0) return 0;
-
-  // pages per numa node
-  int numa_count = (numa_nodes > 0 && numa_nodes <= INT_MAX ? (int)numa_nodes : _mi_os_numa_node_count());
-  if (numa_count <= 0) { numa_count = 1; }
-  const size_t pages_per = pages / numa_count;
-  const size_t pages_mod = pages % numa_count;
-  const size_t timeout_per = (timeout_msecs==0 ? 0 : (timeout_msecs / numa_count) + 50);
-
-  // reserve evenly among numa nodes
-  for (int numa_node = 0; numa_node < numa_count && pages > 0; numa_node++) {
-    size_t node_pages = pages_per;  // can be 0
-    if ((size_t)numa_node < pages_mod) { node_pages++; }
-    int err = mi_reserve_huge_os_pages_at(node_pages, numa_node, timeout_per);
-    if (err) return err;
-    if (pages < node_pages) {
-      pages = 0;
-    }
-    else {
-      pages -= node_pages;
-    }
-  }
-
-  return 0;
-}
-
-int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept {
-  MI_UNUSED(max_secs);
-  _mi_warning_message("mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\n");
-  if (pages_reserved != NULL) *pages_reserved = 0;
-  int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0));
-  if (err==0 && pages_reserved!=NULL) *pages_reserved = pages;
-  return err;
-}
-
-
-
-
-
-/* -----------------------------------------------------------
-  Arena purge
------------------------------------------------------------ */
-
-static long mi_arena_purge_delay(void) {
-  // <0 = no purging allowed, 0=immediate purging, >0=milli-second delay
-  return (mi_option_get(mi_option_purge_delay) * mi_option_get(mi_option_arena_purge_mult));
-}
-
-// reset or decommit in an arena and update the commit bitmap
-// assumes we own the area (i.e. slices_free is claimed by us)
-// returns if the memory is no longer committed (versus reset which keeps the commit)
-static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) {
-  mi_assert_internal(!arena->memid.is_pinned);
-  mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
-
-  const size_t size = mi_size_of_slices(slice_count);
-  void* const p = mi_arena_slice_start(arena, slice_index);
-  //const bool all_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);
-  size_t already_committed;
-  mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..)
-  const bool all_committed = (already_committed == slice_count);
-  const bool needs_recommit = _mi_os_purge_ex(p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg);
-
-  if (needs_recommit) {
-    // no longer committed
-    mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
-    // we just counted in the purge to decommit all, but the some part was not committed so adjust that here
-    // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(slice_count - already_committed));
-  }
-  else if (!all_committed) {
-    // we cannot assume any of these are committed any longer (even with reset since we did setN and may have marked uncommitted slices as committed)
-    mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
-    // we adjust the commit count as parts will be re-committed
-    // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed));
-  }
-
-  return needs_recommit;
-}
-
-
-// Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls.
-// Note: assumes we (still) own the area as we may purge immediately
-static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) {
-  const long delay = mi_arena_purge_delay();
-  if (arena->memid.is_pinned || delay < 0 || _mi_preloading()) return;  // is purging allowed at all?
-
-  mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
-  if (delay == 0) {
-    // purge directly
-    mi_arena_purge(arena, slice_index, slice_count);
-  }
-  else {
-    // schedule purge
-    const mi_msecs_t expire = _mi_clock_now() + delay;
-    mi_msecs_t expire0 = 0;
-    if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) {
-      // expiration was not yet set
-      // maybe set the global arenas expire as well (if it wasn't set already)
-      mi_assert_internal(expire0==0);
-      mi_atomic_casi64_strong_acq_rel(&arena->subproc->purge_expire, &expire0, expire);
-    }
-    else {
-      // already an expiration was set
-    }
-    mi_bitmap_setN(arena->slices_purge, slice_index, slice_count, NULL);
-  }
-}
-
-typedef struct mi_purge_visit_info_s {
-  mi_msecs_t now;
-  mi_msecs_t delay;
-  bool all_purged;
-  bool any_purged;
-} mi_purge_visit_info_t;
-
-static bool mi_arena_try_purge_range(mi_arena_t* arena, size_t slice_index, size_t slice_count) {
-  mi_assert(slice_count < MI_BCHUNK_BITS);
-  if (mi_bbitmap_try_clearNC(arena->slices_free, slice_index, slice_count)) {
-    // purge
-    bool decommitted = mi_arena_purge(arena, slice_index, slice_count); MI_UNUSED(decommitted);
-    mi_assert_internal(!decommitted || mi_bitmap_is_clearN(arena->slices_committed, slice_index, slice_count));
-    // and reset the free range
-    mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);
-    return true;
-  }
-  else {
-    // was allocated again already
-    return false;
-  }
-}
-
-static bool mi_arena_try_purge_visitor(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) {
-  mi_purge_visit_info_t* vinfo = (mi_purge_visit_info_t*)arg;
-  // try to purge: first claim the free blocks
-  if (mi_arena_try_purge_range(arena, slice_index, slice_count)) {
-    vinfo->any_purged = true;
-    vinfo->all_purged = true;
-  }
-  else if (slice_count > 1)
-  {
-    // failed to claim the full range, try per slice instead
-    for (size_t i = 0; i < slice_count; i++) {
-      const bool purged = mi_arena_try_purge_range(arena, slice_index + i, 1);
-      vinfo->any_purged = vinfo->any_purged || purged;
-      vinfo->all_purged = vinfo->all_purged && purged;
-    }
-  }
-  // don't clear the purge bits as that is done atomically be the _bitmap_forall_set_ranges
-  // mi_bitmap_clearN(arena->slices_purge, slice_index, slice_count);
-  return true; // continue
-}
-
-// returns
-// -1 = nothing was purged
-// 0  = nothing was purged yet because have not yet reached the expire time
-// 1  = some pages in the arena were purged
-static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
-{
-  // check pre-conditions
-  if (arena->memid.is_pinned) return -1;
-
-  // expired yet?
-  mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);
-  if (!force) {
-    if (expire==0) return -1;
-    if (expire > now) return 0;
-  }
-
-  // reset expire
-  mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0);
-  mi_subproc_stat_counter_increase(arena->subproc, arena_purges, 1);
-
-  // go through all purge info's  (with max MI_BFIELD_BITS ranges at a time)
-  // this also clears those ranges atomically (so any newly freed blocks will get purged next
-  // time around)
-  mi_purge_visit_info_t vinfo = { now, mi_arena_purge_delay(), true /*all?*/, false /*any?*/};
-
-  // we purge by at least `minslices` to not fragment transparent huge pages for example
-  const size_t minslices = mi_slice_count_of_size(_mi_os_minimal_purge_size());
-  _mi_bitmap_forall_setc_rangesn(arena->slices_purge, minslices, &mi_arena_try_purge_visitor, arena, &vinfo);
-
-  return (vinfo.any_purged ? 1 : -1);
-}
-
-
-static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq)
-{
-  // try purge can be called often so try to only run when needed
-  const long delay = mi_arena_purge_delay();
-  if (_mi_preloading() || delay <= 0) return;  // nothing will be scheduled
-
-  // check if any arena needs purging?
-  const mi_msecs_t now = _mi_clock_now();
-  const mi_msecs_t arenas_expire = mi_atomic_loadi64_acquire(&subproc->purge_expire);
-  if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return;
-
-  const size_t max_arena = mi_arenas_get_count(subproc);
-  if (max_arena == 0) return;
-
-  // allow only one thread to purge at a time (todo: allow concurrent purging?)
-  static mi_atomic_guard_t purge_guard;
-  mi_atomic_guard(&purge_guard)
-  {
-    // increase global expire: at most one purge per delay cycle
-    if (arenas_expire > now) { mi_atomic_storei64_release(&subproc->purge_expire, now + (delay/10)); }
-    const size_t arena_start = tseq % max_arena;
-    size_t max_purge_count = (visit_all ? max_arena : (max_arena/4)+1);
-    bool all_visited = true;
-    bool any_purged = false;
-    for (size_t _i = 0; _i < max_arena; _i++) {
-      size_t i = _i + arena_start;
-      if (i >= max_arena) { i -= max_arena; }
-      mi_arena_t* arena = mi_arena_from_index(subproc,i);
-      if (arena != NULL) {
-        const int purged = mi_arena_try_purge(arena, now, force);
-        if (purged >= 0) {      // purged, or arena expire is not yet reached
-          any_purged = true;
-          if (purged >= 1) {    // purged
-            if (max_purge_count <= 1) {
-              all_visited = false;
-              break;
-            }
-            max_purge_count--;
-          }
-        }
-      }
-    }
-    if (all_visited && !any_purged) {
-      mi_atomic_storei64_release(&subproc->purge_expire, (mi_msecs_t)0);
-    }
-  }
-}
-
-
-/* -----------------------------------------------------------
-  Visit all pages and blocks in a heap
------------------------------------------------------------ */
-
-typedef struct mi_heap_visit_info_s {
-  mi_heap_t* heap;
-  mi_block_visit_fun* visitor;
-  void* arg;
-  bool visit_blocks;
-} mi_heap_visit_info_t;
-
-static bool mi_heap_visit_page(mi_page_t* page, mi_heap_visit_info_t* vinfo) {
-  mi_heap_area_t area;
-  _mi_heap_area_init(&area, page);
-  mi_assert_internal(vinfo->heap == mi_page_heap(page));
-  if (!vinfo->visitor(vinfo->heap, &area, NULL, area.block_size, vinfo->arg)) {
-    return false;
-  }
-  if (vinfo->visit_blocks) {
-    return _mi_theap_area_visit_blocks(&area, page, vinfo->visitor, vinfo->arg);
-  }
-  else {
-    return true;
-  }
-}
-
-static bool mi_heap_visit_page_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) {
-  MI_UNUSED(slice_count);
-  mi_heap_visit_info_t* vinfo = (mi_heap_visit_info_t*)arg;
-  mi_page_t* page = mi_arena_page_at_slice(arena, slice_index);
-  return mi_heap_visit_page(page, vinfo);
-}
-
-bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
-  mi_assert(visitor!=NULL);
-  if (visitor==NULL) return false;
-  if (heap==NULL) { heap = mi_heap_main(); }
-  // visit all pages in a heap
-  // we don't have to claim because we assume we are the only thread running (with this heap).
-  // (but we could atomically claim as well by first doing abandoned_reclaim and afterwards reabandoning).
-  mi_heap_visit_info_t visit_info = { heap, visitor, arg, visit_blocks };
-  bool ok = true;
-  mi_forall_arenas(heap, NULL, 0, arena) {
-    mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);
-    if (ok && arena_pages != NULL) {
-      if (abandoned_only) {
-        for (size_t bin = 0; ok && bin < MI_BIN_COUNT; bin++) {
-          // todo: if we had a single abandoned page map as well, this can be faster.
-          if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) {
-            ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info);
-          }
-        }
-      }
-      else {
-        ok = _mi_bitmap_forall_set(arena_pages->pages, &mi_heap_visit_page_at, arena, &visit_info);
-      }
-    }
-  }
-  mi_forall_arenas_end();
-  if (!ok) return false;
-
-  // visit abandoned pages in OS allocated memory
-  // (technically we don't need the initial lock as we assume we are the only thread running in this subproc)
-  mi_page_t* page = NULL;
-  mi_lock(&heap->os_abandoned_pages_lock) {
-    page = heap->os_abandoned_pages;
-  }
-  while (ok && page != NULL) {
-    mi_page_t* next = page->next;  // read upfront in case the visitor frees the page
-    ok = mi_heap_visit_page(page, &visit_info);
-    page = next;
-  }
-
-  return ok;
-}
-
-bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
-  return _mi_heap_visit_blocks(heap, false, visit_blocks, visitor, arg);
-}
-
-bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
-  return _mi_heap_visit_blocks(heap, true, visit_blocks, visitor, arg);
-}
-
-
-typedef struct mi_heap_delete_visit_info_s {
-  mi_heap_t*  heap_target;
-  mi_theap_t* theap_target;
-  mi_theap_t* theap;
-} mi_heap_delete_visit_info_t;
-
-static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) {
-  MI_UNUSED(block); MI_UNUSED(block_size); MI_UNUSED(heap);
-  mi_heap_delete_visit_info_t* info = (mi_heap_delete_visit_info_t*)arg;
-  mi_heap_t*  heap_target           = info->heap_target;
-  mi_theap_t* const theap           = NULL; // info->theap;       mi_assert_internal(_mi_theap_heap(theap) == heap);
-  mi_page_t*  const page            = (mi_page_t*)area->reserved1;
-
-  mi_page_claim_ownership(page);       // claim ownership
-  if (mi_page_is_abandoned(page)) {
-    _mi_arenas_page_unabandon(page,theap);
-  }
-  else {
-    page->next = page->prev = NULL;    // yikes.. better not to try to access this from a thread later on..
-    mi_page_set_theap(page,NULL);      // set threadid to abandoned
-  }
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(mi_page_is_owned(page));
-
-  if (page->used==0) {
-    // free the page
-    _mi_arenas_page_free(page, theap);
-  }
-  else if (heap_target==NULL) {
-    // destroy the page
-    page->used=0;                        // note: invariant `|local_free| + |free| == reserved - used`  does not hold in this case
-    _mi_arenas_page_free(page, theap);
-  }
-  else {
-    // move the page to `heap_target` as an abandoned page
-    // first remove it from the current heap
-    const size_t sbin = _mi_page_stats_bin(page);
-    size_t slice_index;
-    size_t slice_count;
-    mi_arena_pages_t* arena_pages = NULL;
-    mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
-    mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index));
-    mi_bitmap_clear(arena_pages->pages, slice_index);
-    if (theap != NULL) {
-      mi_theap_stat_decrease(theap, page_bins[sbin], 1);
-      mi_theap_stat_decrease(theap, pages, 1);
-    }
-    else {
-      mi_heap_stat_decrease((mi_heap_t*)heap, page_bins[_mi_page_stats_bin(page)], 1);
-      mi_heap_stat_decrease((mi_heap_t*)heap, pages, 1);
-    }
-    mi_theap_t* theap_target = info->theap_target;
-
-    // and then add it to the new target heap
-    mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);
-    if mi_unlikely(arena_pages_target==NULL) {
-      // if we cannot allocate this, we move it to the main heap instead (which does not require allocation)
-      heap_target = mi_heap_main();
-      theap_target = mi_heap_theap(heap_target);
-      arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);
-      mi_assert_internal(arena_pages_target!=NULL);
-    }
-    mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index));
-    mi_bitmap_set(arena_pages_target->pages, slice_index);
-    page->heap = heap_target;
-    mi_theap_stat_increase(theap_target, page_bins[sbin], 1);
-    mi_theap_stat_increase(theap_target, pages, 1);
-
-    // and abandon in the new heap
-    _mi_arenas_page_abandon(page,theap_target);
-  }
-  return true;
-}
-
-static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) {
-  mi_theap_t* const theap_target = (heap_target != NULL ? _mi_heap_theap(heap_target) : NULL);
-  // mi_theap_t* const theap = _mi_heap_theap(heap);
-  mi_heap_delete_visit_info_t info = { heap_target, theap_target, NULL };
-  _mi_heap_visit_blocks(heap, false, false, &mi_heap_delete_page, &info);
-  #if MI_DEBUG>1
-  // no more arena pages?
-  for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
-    mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
-    if (arena_pages!=NULL) {
-      mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages));
-    }
-  }
-  // nor os abandoned pages?
-  mi_lock(&heap->os_abandoned_pages_lock) {
-
-    mi_assert_internal(heap->os_abandoned_pages == NULL);
-  }
-  // nor arena abandoned pages?
-  for (size_t i = 0; i < MI_BIN_COUNT; i++) {
-    mi_assert_internal(mi_atomic_load_relaxed(&heap->abandoned_count[i])==0);
-  }
-  #endif
-}
-
-void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to) {
-  if (_mi_is_heap_main(heap_from)) return;
-  if (heap_to==NULL) { heap_to = mi_heap_main(); }
-  mi_heap_delete_pages(heap_from, heap_to);
-}
-
-void _mi_heap_destroy_pages(mi_heap_t* heap_from) {
-  if (_mi_is_heap_main(heap_from)) return;
-  mi_heap_delete_pages(heap_from, NULL);
-}
-
-/* -----------------------------------------------------------
-  Unloading and reloading an arena.
------------------------------------------------------------ */
-/*
-static bool mi_arena_page_register(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) {
-  MI_UNUSED(arg); MI_UNUSED(slice_count);
-  mi_assert_internal(slice_count == 1);
-  mi_page_t* page = mi_arena_page_at_slice(arena, slice_index);
-  mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1));
-  if (!_mi_page_map_register(page)) return false; // break
-  mi_assert_internal(_mi_ptr_page(page)==page);
-  return true;
-}
-
-mi_decl_nodiscard static bool mi_arena_pages_reregister(mi_arena_t* arena) {
-  return _mi_bitmap_forall_set(arena->pages, &mi_arena_page_register, arena, NULL);
-}
-
-mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* full_size) {
-  mi_arena_t* arena = _mi_arena_from_id(arena_id);
-  if (arena==NULL) {
-    return false;
-  }
-  else if (!arena->is_exclusive) {
-    _mi_warning_message("cannot unload a non-exclusive arena (id %zu at %p)\n", arena_id, arena);
-    return false;
-  }
-  else if (arena->memid.memkind != MI_MEM_EXTERNAL) {
-    _mi_warning_message("can only unload managed arena's for external memory (id %zu at %p)\n", arena_id, arena);
-    return false;
-  }
-
-  // find accessed size
-  const size_t asize = mi_size_of_slices(mi_arena_used_slices(arena));
-  if (base != NULL) { *base = (void*)arena; }
-  if (full_size != NULL) { *full_size = arena->memid.mem.os.size;  }
-  if (accessed_size != NULL) { *accessed_size = asize; }
-
-  // adjust abandoned page count
-  mi_subproc_t* const subproc = arena->subproc;
-  for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
-    const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);
-    if (count > 0) { mi_atomic_decrement_acq_rel(&subproc->abandoned_count[bin]); }
-  }
-
-  // unregister the pages
-  _mi_page_map_unregister_range(arena, asize);
-
-  // set arena entry to NULL
-  const size_t count = mi_arenas_get_count(subproc);
-  for(size_t i = 0; i < count; i++) {
-    if (mi_arena_from_index(subproc, i) == arena) {
-      mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);
-      if (i + 1 == count) { // try adjust the count?
-        size_t expected = count;
-        mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, count-1);
-      }
-      break;
-    }
-  }
-  return true;
-}
-
-mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) {
-  // assume the memory area is already containing the arena
-  if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }
-  if (start == NULL || size == 0) return false;
-  mi_arena_t* arena = (mi_arena_t*)start;
-  mi_memid_t memid = arena->memid;
-  if (memid.memkind != MI_MEM_EXTERNAL) {
-    _mi_warning_message("can only reload arena's from external memory (%p)\n", arena);
-    return false;
-  }
-  if (memid.mem.os.base != start) {
-    _mi_warning_message("the reloaded arena base address differs from the external memory (arena: %p, external: %p)\n", arena, start);
-    return false;
-  }
-  if (memid.mem.os.size != size) {
-    _mi_warning_message("the reloaded arena size differs from the external memory (arena size: %zu, external size: %zu)\n", arena->memid.mem.os.size, size);
-    return false;
-  }
-  if (!arena->is_exclusive) {
-    _mi_warning_message("the reloaded arena is not exclusive\n");
-    return false;
-  }
-
-  // re-initialize
-  arena->is_exclusive = true;
-  arena->commit_fun = commit_fun;
-  arena->commit_fun_arg = commit_fun_arg;
-  arena->subproc = _mi_subproc();
-  if (!mi_arenas_add(arena->subproc, arena, arena_id)) {
-    return false;
-  }
-  if (!mi_arena_pages_reregister(arena)) {
-    // todo: clear arena entry in the subproc?
-    return false;
-  }
-
-  // adjust abandoned page count
-  for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
-    const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);
-    if (count > 0) { mi_atomic_decrement_acq_rel(&arena->subproc->abandoned_count[bin]); }
-  }
-
-  return true;
-}
-
-*/
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.c
deleted file mode 100644
index 67269e8b..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.c
+++ /dev/null
@@ -1,1949 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* ----------------------------------------------------------------------------
-Concurrent bitmap that can set/reset sequences of bits atomically
----------------------------------------------------------------------------- */
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/bits.h"
-#include "mimalloc/prim.h"  // _mi_prim_thread_yield
-#include "bitmap.h"
-
-#ifndef MI_OPT_SIMD
-#define MI_OPT_SIMD   0
-#endif
-
-/* --------------------------------------------------------------------------------
-  bfields
--------------------------------------------------------------------------------- */
-
-static inline size_t mi_bfield_ctz(mi_bfield_t x) {
-  return mi_ctz(x);
-}
-
-static inline size_t mi_bfield_clz(mi_bfield_t x) {
-  return mi_clz(x);
-}
-
-static inline size_t mi_bfield_popcount(mi_bfield_t x) {
-  return mi_popcount(x);
-}
-
-static inline mi_bfield_t mi_bfield_clear_least_bit(mi_bfield_t x) {
-  return (x & (x-1));
-}
-
-// find the least significant bit that is set (i.e. count trailing zero's)
-// return false if `x==0` (with `*idx` undefined) and true otherwise,
-// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
-static inline bool mi_bfield_find_least_bit(mi_bfield_t x, size_t* idx) {
-  return mi_bsf(x,idx);
-}
-
-// find the most significant bit that is set.
-// return false if `x==0` (with `*idx` undefined) and true otherwise,
-// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
-static inline bool mi_bfield_find_highest_bit(mi_bfield_t x, size_t* idx) {
-  return mi_bsr(x, idx);
-}
-
-
-
-// find each set bit in a bit field `x` and clear it, until it becomes zero.
-static inline bool mi_bfield_foreach_bit(mi_bfield_t* x, size_t* idx) {
-  const bool found = mi_bfield_find_least_bit(*x, idx);
-  *x = mi_bfield_clear_least_bit(*x);
-  return found;
-}
-
-static inline mi_bfield_t mi_bfield_zero(void) {
-  return 0;
-}
-
-static inline mi_bfield_t mi_bfield_one(void) {
-  return 1;
-}
-
-static inline mi_bfield_t mi_bfield_all_set(void) {
-  return ~((mi_bfield_t)0);
-}
-
-// mask of `bit_count` bits set shifted to the left by `shiftl`
-static inline mi_bfield_t mi_bfield_mask(size_t bit_count, size_t shiftl) {
-  mi_assert_internal(bit_count > 0);
-  mi_assert_internal(bit_count + shiftl <= MI_BFIELD_BITS);
-  mi_assert_internal(shiftl < MI_BFIELD_BITS);
-  const mi_bfield_t mask0 = (bit_count < MI_BFIELD_BITS ? (mi_bfield_one() << bit_count)-1 : mi_bfield_all_set());
-  return (mask0 << shiftl);
-}
-
-
-// ------- mi_bfield_atomic_set ---------------------------------------
-// the `_set` functions return also the count of bits that were already set (for commit statistics)
-// the `_clear` functions return also whether the new bfield is all clear or not (for the chunk_map)
-
-// Set a bit atomically. Returns `true` if the bit transitioned from 0 to 1
-static inline bool mi_bfield_atomic_set(_Atomic(mi_bfield_t)*b, size_t idx) {
-  mi_assert_internal(idx < MI_BFIELD_BITS);
-  const mi_bfield_t mask = mi_bfield_mask(1, idx);;
-  const mi_bfield_t old = mi_atomic_or_acq_rel(b, mask);
-  return ((old&mask) == 0);
-}
-
-// Clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0.
-// `all_clear` is set if the new bfield is zero.
-static inline bool mi_bfield_atomic_clear(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) {
-  mi_assert_internal(idx < MI_BFIELD_BITS);
-  const mi_bfield_t mask = mi_bfield_mask(1, idx);;
-  mi_bfield_t old = mi_atomic_and_acq_rel(b, ~mask);
-  if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }
-  return ((old&mask) == mask);
-}
-
-// Clear a bit but only when/once it is set. This is used by concurrent free's while
-// the page is abandoned and mapped. This can incure a busy wait :-( but it should
-// happen almost never (and is accounted for in the stats)
-static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_t idx) {
-  mi_assert_internal(idx < MI_BFIELD_BITS);
-  const mi_bfield_t mask = mi_bfield_mask(1, idx);;
-  mi_bfield_t old = mi_atomic_load_relaxed(b);
-  do {
-    if mi_unlikely((old&mask) == 0) {
-      old = mi_atomic_load_acquire(b);
-      if ((old&mask)==0) {
-        mi_subproc_stat_counter_increase(_mi_subproc(), pages_unabandon_busy_wait, 1);
-      }
-      while ((old&mask)==0) { // busy wait
-        _mi_prim_thread_yield(); 
-        old = mi_atomic_load_acquire(b);
-      }
-    }
-  } while (!mi_atomic_cas_weak_acq_rel(b,&old, (old&~mask)));
-  mi_assert_internal((old&mask)==mask);  // we should only clear when it was set
-}
-
-// Set a mask set of bits atomically, and return true of the mask bits transitioned from all 0's to 1's.
-// `already_set` contains the count of bits that were already set (used when committing ranges to account
-// statistics correctly).
-static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) {
-  mi_assert_internal(mask != 0);
-  mi_bfield_t old = mi_atomic_load_relaxed(b);
-  while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) {};  // try to atomically set the mask bits until success
-  if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); }
-  return ((old&mask) == 0);
-}
-
-// Clear a mask set of bits atomically, and return true of the mask bits transitioned from all 1's to 0's
-// `all_clear` is set to `true` if the new bfield became zero.
-static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, bool* all_clear) {
-  mi_assert_internal(mask != 0);
-  mi_bfield_t old = mi_atomic_load_relaxed(b);
-  while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) {};  // try to atomically clear the mask bits until success
-  if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }
-  return ((old&mask) == mask);
-}
-
-static inline bool mi_bfield_atomic_setX(_Atomic(mi_bfield_t)*b, size_t* already_set) {
-  const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_all_set());
-  if (already_set!=NULL) { *already_set = mi_bfield_popcount(old); }
-  return (old==0);
-}
-
-// static inline bool mi_bfield_atomic_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) {
-//   const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_zero());
-//   if (all_clear!=NULL) { *all_clear = true; }
-//   return (~old==0);
-// }
-
-// ------- mi_bfield_atomic_try_clear ---------------------------------------
-
-
-// Tries to clear a mask atomically, and returns true if the mask bits atomically transitioned from mask to 0
-// and false otherwise (leaving the bit field as is).
-// `all_clear` is set to `true` if the new bfield became zero.
-static inline bool mi_bfield_atomic_try_clear_mask_of(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t expect, bool* all_clear) {
-  mi_assert_internal(mask != 0);
-  // try to atomically clear the mask bits
-  do {
-    if ((expect & mask) != mask) {  // are all bits still set?
-      if (all_clear != NULL) { *all_clear = (expect == 0); }
-      return false;
-    }
-  } while (!mi_atomic_cas_weak_acq_rel(b, &expect, expect & ~mask));
-  if (all_clear != NULL) { *all_clear = ((expect & ~mask) == 0);  }
-  return true;
-}
-
-static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear) {
-  mi_assert_internal(mask != 0);
-  const mi_bfield_t expect = mi_atomic_load_relaxed(b);
-  return mi_bfield_atomic_try_clear_mask_of(b, mask, expect, all_clear);
-}
-
-// Tries to clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0
-// and `false` otherwise leaving the bfield `b` as-is.
-// `all_clear` is set to true if the new bfield became zero (and false otherwise)
-mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear(_Atomic(mi_bfield_t)* b, size_t idx, bool* all_clear) {
-  mi_assert_internal(idx < MI_BFIELD_BITS);
-  const mi_bfield_t mask = mi_bfield_one()<<idx;
-  return mi_bfield_atomic_try_clear_mask(b, mask, all_clear);
-}
-
-// Tries to clear a byte atomically, and returns true if the byte atomically transitioned from 0xFF to 0
-// `all_clear` is set to true if the new bfield became zero (and false otherwise)
-mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear8(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) {
-  mi_assert_internal(idx < MI_BFIELD_BITS);
-  mi_assert_internal((idx%8)==0);
-  const mi_bfield_t mask = ((mi_bfield_t)0xFF)<<idx;
-  return mi_bfield_atomic_try_clear_mask(b, mask, all_clear);
-}
-
-// Try to clear a full field of bits atomically, and return true all bits transitioned from all 1's to 0's.
-// and false otherwise leaving the bit field as-is.
-// `all_clear` is set to true if the new bfield became zero (which is always the case if successful).
-static inline bool mi_bfield_atomic_try_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) {
-  mi_bfield_t old = mi_bfield_all_set();
-  if (mi_atomic_cas_strong_acq_rel(b, &old, mi_bfield_zero())) {
-    if (all_clear != NULL) { *all_clear = true; }
-    return true;
-  }
-  else return false;
-}
-
-
-// ------- mi_bfield_atomic_is_set ---------------------------------------
-
-// Check if a bit is set
-static inline bool mi_bfield_atomic_is_set(const _Atomic(mi_bfield_t)*b, const size_t idx) {
-  const mi_bfield_t x = mi_atomic_load_acquire(b);
-  return ((x & mi_bfield_mask(1,idx)) != 0);
-}
-
-// Check if a bit is clear
-static inline bool mi_bfield_atomic_is_clear(const _Atomic(mi_bfield_t)*b, const size_t idx) {
-  const mi_bfield_t x = mi_atomic_load_acquire(b);
-  return ((x & mi_bfield_mask(1, idx)) == 0);
-}
-
-// Check if a bit is xset
-static inline bool mi_bfield_atomic_is_xset(mi_xset_t set, const _Atomic(mi_bfield_t)*b, const size_t idx) {
-  if (set) return mi_bfield_atomic_is_set(b, idx);
-      else return mi_bfield_atomic_is_clear(b, idx);
-}
-
-// Check if all bits corresponding to a mask are set.
-static inline bool mi_bfield_atomic_is_set_mask(const _Atomic(mi_bfield_t)* b, mi_bfield_t mask) {
-  mi_assert_internal(mask != 0);
-  const mi_bfield_t x = mi_atomic_load_acquire(b);
-  return ((x & mask) == mask);
-}
-
-// Check if all bits corresponding to a mask are clear.
-static inline bool mi_bfield_atomic_is_clear_mask(const _Atomic(mi_bfield_t)* b, mi_bfield_t mask) {
-  mi_assert_internal(mask != 0);
-  const mi_bfield_t x = mi_atomic_load_acquire(b);
-  return ((x & mask) == 0);
-}
-
-// Check if all bits corresponding to a mask are set/cleared.
-static inline bool mi_bfield_atomic_is_xset_mask(mi_xset_t set, const _Atomic(mi_bfield_t)* b, mi_bfield_t mask) {
-  mi_assert_internal(mask != 0);
-  if (set) return mi_bfield_atomic_is_set_mask(b, mask);
-      else return mi_bfield_atomic_is_clear_mask(b, mask);
-}
-
-// Count bits in a mask
-static inline size_t mi_bfield_atomic_popcount_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask) {
-  const mi_bfield_t x = mi_atomic_load_acquire(b);
-  return mi_bfield_popcount(x & mask);
-}
-
-
-/* --------------------------------------------------------------------------------
- bitmap chunks
--------------------------------------------------------------------------------- */
-
-// ------- mi_bchunk_set ---------------------------------------
-
-// Set a single bit
-static inline bool mi_bchunk_set(mi_bchunk_t* chunk, size_t cidx, size_t* already_set) {
-  mi_assert_internal(cidx < MI_BCHUNK_BITS);
-  const size_t i = cidx / MI_BFIELD_BITS;
-  const size_t idx = cidx % MI_BFIELD_BITS;
-  const bool was_clear = mi_bfield_atomic_set(&chunk->bfields[i], idx);
-  if (already_set != NULL) { *already_set = (was_clear ? 0 : 1); }
-  return was_clear;
-}
-
-// Set `0 < n <= MI_BFIELD_BITS`, and return true of the mask bits transitioned from all 0's to 1's.
-// `already_set` contains the count of bits that were already set (used when committing ranges to account
-// statistics correctly).
-// Can cross over two bfields.
-static inline bool mi_bchunk_setNX(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) {
-  mi_assert_internal(cidx < MI_BCHUNK_BITS);
-  mi_assert_internal(n > 0 && n <= MI_BFIELD_BITS);
-  const size_t i = cidx / MI_BFIELD_BITS;
-  const size_t idx = cidx % MI_BFIELD_BITS;
-  if mi_likely(idx + n <= MI_BFIELD_BITS) {
-    // within one field
-    return mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(n,idx), already_set);
-  }
-  else {
-    // spanning two fields
-    const size_t m = MI_BFIELD_BITS - idx;  // bits to clear in the first field
-    mi_assert_internal(m < n);
-    mi_assert_internal(i < MI_BCHUNK_FIELDS - 1);
-    mi_assert_internal(idx + m <= MI_BFIELD_BITS);
-    size_t already_set1;
-    const bool all_set1 = mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &already_set1);
-    mi_assert_internal(n - m > 0);
-    mi_assert_internal(n - m < MI_BFIELD_BITS);
-    size_t already_set2;
-    const bool all_set2 = mi_bfield_atomic_set_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &already_set2);
-    if (already_set != NULL) { *already_set = already_set1 + already_set2; }
-    return (all_set1 && all_set2);
-  }
-}
-
-// Set a sequence of `n` bits within a chunk.
-// Returns true if all bits transitioned from 0 to 1 (or 1 to 0).
-mi_decl_noinline static bool mi_bchunk_xsetNC(mi_xset_t set, mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* palready_set, bool* pmaybe_all_clear) {
-  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
-  mi_assert_internal(n>0);
-  bool all_transition = true;
-  bool maybe_all_clear = true;
-  size_t total_already_set = 0;
-  size_t idx   = cidx % MI_BFIELD_BITS;
-  size_t field = cidx / MI_BFIELD_BITS;
-  while (n > 0) {
-    size_t m = MI_BFIELD_BITS - idx;   // m is the bits to xset in this field
-    if (m > n) { m = n; }
-    mi_assert_internal(idx + m <= MI_BFIELD_BITS);
-    mi_assert_internal(field < MI_BCHUNK_FIELDS);
-    const mi_bfield_t mask = mi_bfield_mask(m, idx);
-    size_t already_set = 0;
-    bool all_clear = false;
-    const bool transition = (set ? mi_bfield_atomic_set_mask(&chunk->bfields[field], mask, &already_set)
-                                 : mi_bfield_atomic_clear_mask(&chunk->bfields[field], mask, &all_clear));
-    mi_assert_internal((transition && already_set == 0) || (!transition && already_set > 0));
-    all_transition = all_transition && transition;
-    total_already_set += already_set;
-    maybe_all_clear = maybe_all_clear && all_clear;
-    // next field
-    field++;
-    idx = 0;
-    mi_assert_internal(m <= n);
-    n -= m;
-  }
-  if (palready_set!=NULL) { *palready_set = total_already_set; }
-  if (pmaybe_all_clear!=NULL) { *pmaybe_all_clear = maybe_all_clear; }
-  return all_transition;
-}
-
-static inline bool mi_bchunk_setN(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) {
-  mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS);
-  if (n==1) return mi_bchunk_set(chunk, cidx, already_set);
-  // if (n==8 && (cidx%8) == 0) return mi_bchunk_set8(chunk, cidx, already_set);
-  // if (n==MI_BFIELD_BITS) return mi_bchunk_setX(chunk, cidx, already_set);
-  if (n<=MI_BFIELD_BITS) return mi_bchunk_setNX(chunk, cidx, n, already_set);
-  return mi_bchunk_xsetNC(MI_BIT_SET, chunk, cidx, n, already_set, NULL);
-}
-
-// ------- mi_bchunk_clear ---------------------------------------
-
-static inline bool mi_bchunk_clear(mi_bchunk_t* chunk, size_t cidx, bool* all_clear) {
-  mi_assert_internal(cidx < MI_BCHUNK_BITS);
-  const size_t i = cidx / MI_BFIELD_BITS;
-  const size_t idx = cidx % MI_BFIELD_BITS;
-  return mi_bfield_atomic_clear(&chunk->bfields[i], idx, all_clear);
-}
-
-static inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) {
-  mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS);
-  if (n==1) return mi_bchunk_clear(chunk, cidx, maybe_all_clear);
-  // if (n==8) return mi_bchunk_clear8(chunk, cidx, maybe_all_clear);
-  // if (n==MI_BFIELD_BITS) return mi_bchunk_clearX(chunk, cidx, maybe_all_clear);
-  // TODO: implement mi_bchunk_xsetNX instead of setNX
-  return mi_bchunk_xsetNC(MI_BIT_CLEAR, chunk, cidx, n, NULL, maybe_all_clear);
-}
-
-// Check if a sequence of `n` bits within a chunk are all set/cleared.
-// This can cross bfield's
-mi_decl_noinline static size_t mi_bchunk_popcountNC(mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) {
-  mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS);
-  size_t count = 0;
-  while (n > 0) {
-    size_t m = MI_BFIELD_BITS - idx;   // m is the bits to xset in this field
-    if (m > n) { m = n; }
-    mi_assert_internal(idx + m <= MI_BFIELD_BITS);
-    mi_assert_internal(field_idx < MI_BCHUNK_FIELDS);
-    const size_t mask = mi_bfield_mask(m, idx);
-    count += mi_bfield_atomic_popcount_mask(&chunk->bfields[field_idx], mask);
-    // next field
-    field_idx++;
-    idx = 0;
-    n -= m;
-  }
-  return count;
-}
-
-// Count set bits a sequence of `n` bits.
-static inline size_t mi_bchunk_popcountN(mi_bchunk_t* chunk, size_t cidx, size_t n) {
-  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
-  mi_assert_internal(n>0);
-  if (n==0) return 0;
-  const size_t i = cidx / MI_BFIELD_BITS;
-  const size_t idx = cidx % MI_BFIELD_BITS;
-  if (n==1) { return (mi_bfield_atomic_is_set(&chunk->bfields[i], idx) ? 1 : 0); }
-  if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_popcount_mask(&chunk->bfields[i], mi_bfield_mask(n, idx)); }
-  return mi_bchunk_popcountNC(chunk, i, idx, n);
-}
-
-
-// ------- mi_bchunk_is_xset ---------------------------------------
-
-// Check if a sequence of `n` bits within a chunk are all set/cleared.
-// This can cross bfield's
-mi_decl_noinline static bool mi_bchunk_is_xsetNC(mi_xset_t set, const mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) {
-  mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS);
-  while (n > 0) {
-    size_t m = MI_BFIELD_BITS - idx;   // m is the bits to xset in this field
-    if (m > n) { m = n; }
-    mi_assert_internal(idx + m <= MI_BFIELD_BITS);
-    mi_assert_internal(field_idx < MI_BCHUNK_FIELDS);
-    const size_t mask = mi_bfield_mask(m, idx);
-    if (!mi_bfield_atomic_is_xset_mask(set, &chunk->bfields[field_idx], mask)) {
-      return false;
-    }
-    // next field
-    field_idx++;
-    idx = 0;
-    n -= m;
-  }
-  return true;
-}
-
-// Check if a sequence of `n` bits within a chunk are all set/cleared.
-static inline bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, size_t cidx, size_t n) {
-  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
-  mi_assert_internal(n>0);
-  if (n==0) return true;
-  const size_t i = cidx / MI_BFIELD_BITS;
-  const size_t idx = cidx % MI_BFIELD_BITS;
-  if (n==1) { return mi_bfield_atomic_is_xset(set, &chunk->bfields[i], idx); }
-  if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_is_xset_mask(set, &chunk->bfields[i], mi_bfield_mask(n, idx)); }
-  return mi_bchunk_is_xsetNC(set, chunk, i, idx, n);
-}
-
-
-// ------- mi_bchunk_try_clear  ---------------------------------------
-
-// Clear `0 < n <= MI_BITFIELD_BITS`. Can cross over a bfield boundary.
-static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {
-  mi_assert_internal(cidx < MI_BCHUNK_BITS);
-  mi_assert_internal(n <= MI_BFIELD_BITS);
-  const size_t i = cidx / MI_BFIELD_BITS;
-  const size_t idx = cidx % MI_BFIELD_BITS;
-  if mi_likely(idx + n <= MI_BFIELD_BITS) {
-    // within one field
-    return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear);
-  }
-  else {
-    // spanning two fields (todo: use double-word atomic ops?)
-    const size_t m = MI_BFIELD_BITS - idx;  // bits to clear in the first field
-    mi_assert_internal(m < n);
-    mi_assert_internal(i < MI_BCHUNK_FIELDS - 1);
-    bool field1_is_clear;
-    if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear)) return false;
-    // try the second field as well
-    mi_assert_internal(n - m > 0);
-    mi_assert_internal(n - m < MI_BFIELD_BITS);
-    bool field2_is_clear;
-    if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear)) {
-      // we failed to clear the second field, restore the first one
-      mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), NULL);
-      return false;
-    }
-    if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = field1_is_clear && field2_is_clear;  }
-    return true;
-  }
-}
-
-// Clear a full aligned bfield.
-// static inline bool mi_bchunk_try_clearX(mi_bchunk_t* chunk, size_t cidx, bool* pmaybe_all_clear) {
-//   mi_assert_internal(cidx < MI_BCHUNK_BITS);
-//   mi_assert_internal((cidx%MI_BFIELD_BITS) == 0);
-//   const size_t i = cidx / MI_BFIELD_BITS;
-//   return mi_bfield_atomic_try_clearX(&chunk->bfields[i], pmaybe_all_clear);
-// }
-
-// Try to atomically clear a sequence of `n` bits within a chunk.
-// Returns true if all bits transitioned from 1 to 0,
-// and false otherwise leaving all bit fields as is.
-// Note: this is the complex one as we need to unwind partial atomic operations if we fail halfway..
-// `maybe_all_clear` is set to `true` if all the bfields involved become zero.
-mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {
-  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
-  mi_assert_internal(n>0);
-  if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = true; }
-  if (n==0) return true;
-
-  // first field
-  const size_t start_idx = cidx % MI_BFIELD_BITS;
-  const size_t start_field = cidx / MI_BFIELD_BITS;
-  size_t field = start_field;
-  size_t m = MI_BFIELD_BITS - start_idx;   // m are the bits to clear in this field
-  if (m > n) { m = n; }
-  mi_assert_internal(start_idx + m <= MI_BFIELD_BITS);
-  mi_assert_internal(start_field < MI_BCHUNK_FIELDS);
-  const mi_bfield_t mask_start = mi_bfield_mask(m, start_idx);
-  bool maybe_all_clear;
-  if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear)) return false;
-
-  // done?
-  mi_assert_internal(m <= n);
-  n -= m;
-
-  // continue with mid fields and last field: if these fail we need to recover by unsetting previous fields
-  // mid fields?
-  while (n >= MI_BFIELD_BITS) {
-    field++;
-    mi_assert_internal(field < MI_BCHUNK_FIELDS);
-    bool field_is_clear;
-    if (!mi_bfield_atomic_try_clearX(&chunk->bfields[field], &field_is_clear)) goto restore;
-    maybe_all_clear = maybe_all_clear && field_is_clear;
-    n -= MI_BFIELD_BITS;
-  }
-
-  // last field?
-  if (n > 0) {
-    mi_assert_internal(n < MI_BFIELD_BITS);
-    field++;
-    mi_assert_internal(field < MI_BCHUNK_FIELDS);
-    const mi_bfield_t mask_end = mi_bfield_mask(n, 0);
-    bool field_is_clear;
-    if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear)) goto restore;
-    maybe_all_clear = maybe_all_clear && field_is_clear;
-  }
-
-  if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = maybe_all_clear; }
-  return true;
-
-restore:
-  // `field` is the index of the field that failed to set atomically; we need to restore all previous fields
-  mi_assert_internal(field > start_field);
-  while( field > start_field) {
-    field--;
-    if (field == start_field) {
-      mi_bfield_atomic_set_mask(&chunk->bfields[field], mask_start, NULL);
-    }
-    else {
-      mi_bfield_atomic_setX(&chunk->bfields[field], NULL);  // mid-field: set all bits again
-    }
-  }
-  return false;
-}
-
-
-static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) {
-  mi_assert_internal(n>0);
-  // if (n==MI_BFIELD_BITS) return mi_bchunk_try_clearX(chunk, cidx, maybe_all_clear);
-  if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear);
-  return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear);
-}
-
-
-// ------- mi_bchunk_try_find_and_clear ---------------------------------------
-
-#if MI_OPT_SIMD && defined(__AVX2__)
-mi_decl_maybe_unused static inline __m256i mi_mm256_zero(void) {
-  return _mm256_setzero_si256();
-}
-mi_decl_maybe_unused static inline __m256i mi_mm256_ones(void) {
-  return _mm256_set1_epi64x(~0);
-}
-mi_decl_maybe_unused static inline bool mi_mm256_is_ones(__m256i vec) {
-  return _mm256_testc_si256(vec, _mm256_cmpeq_epi32(vec, vec));
-}
-mi_decl_maybe_unused static inline bool mi_mm256_is_zero( __m256i vec) {
-  return _mm256_testz_si256(vec,vec);
-}
-#endif
-
-static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {
-  mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);
-  // note: this must be acquire (and not relaxed), or otherwise the AVX code below can loop forever
-  // as the compiler won't reload the registers vec1 and vec2 from memory again.
-  const mi_bfield_t b = mi_atomic_load_acquire(&chunk->bfields[chunk_idx]);
-  size_t idx;
-  if (mi_bfield_find_least_bit(b, &idx)) {           // find the least bit
-    if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], mi_bfield_mask(1,idx), b, NULL)) {  // clear it atomically
-      *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;
-      mi_assert_internal(*pidx < MI_BCHUNK_BITS);
-      return true;
-    }
-  }
-  return false;
-}
-
-// Find least 1-bit in a chunk and try to clear it atomically
-// set `*pidx` to the bit index (0 <= *pidx < MI_BCHUNK_BITS) on success.
-// This is used to find free slices and abandoned pages and should be efficient.
-// todo: try neon version
-static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx) {
-  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
-  for(int tries=0; tries<4; tries++) {   // paranoia: at most 4 tries
-    const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
-    const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF  : 0)
-    const uint32_t mask = ~_mm256_movemask_epi8(vcmp);  // mask of most significant bit of each byte (so each 8 bits are all set or clear)
-    // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)
-    if (mask==0) return false;
-    mi_assert_internal((_tzcnt_u32(mask)%8) == 0); // tzcnt == 0, 8, 16, or 24
-    const size_t chunk_idx = _tzcnt_u32(mask) / 8;
-    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;
-    // try again
-    // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded
-    // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)
-    #if defined(__GNUC__)
-    __asm __volatile ("" : : "g"(chunk) : "memory");
-    #endif
-  }
-  #elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
-  for(int tries=0; tries<4; tries++) {   // paranoia: at most 4 tries
-    size_t chunk_idx = 0;
-    #if 0
-    // one vector at a time
-    __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
-    if (mi_mm256_is_zero(vec)) {
-      chunk_idx += 4;
-      vec = _mm256_load_si256(((const __m256i*)chunk->bfields) + 1);
-    }
-    const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF  : 0)
-    const uint32_t mask = ~_mm256_movemask_epi8(vcmp);  // mask of most significant bit of each byte (so each 8 bits are all set or clear)
-    // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)
-    if (mask==0) return false;
-    mi_assert_internal((_tzcnt_u32(mask)%8) == 0); // tzcnt == 0, 8, 16, or 24
-    chunk_idx += _tzcnt_u32(mask) / 8;
-    #else
-    // a cache line is 64b so we can just as well load all at the same time
-    const __m256i vec1  = _mm256_load_si256((const __m256i*)chunk->bfields);
-    const __m256i vec2  = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);
-    const __m256i cmpv  = mi_mm256_zero();
-    const __m256i vcmp1 = _mm256_cmpeq_epi64(vec1, cmpv); // (elem64 == 0 ? 0xFF  : 0)
-    const __m256i vcmp2 = _mm256_cmpeq_epi64(vec2, cmpv); // (elem64 == 0 ? 0xFF  : 0)
-    const uint32_t mask1 = ~_mm256_movemask_epi8(vcmp1);  // mask of most significant bit of each byte (so each 8 bits are all set or clear)
-    const uint32_t mask2 = ~_mm256_movemask_epi8(vcmp2);  // mask of most significant bit of each byte (so each 8 bits are all set or clear)
-    const uint64_t mask = ((uint64_t)mask2 << 32) | mask1;
-    // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)
-    if (mask==0) return false;
-    mi_assert_internal((_tzcnt_u64(mask)%8) == 0); // tzcnt == 0, 8, 16, 24 , ..
-    chunk_idx = mi_ctz(mask) / 8;
-    #endif
-    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;
-    // try again
-    // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded
-    // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)
-    #if defined(__GNUC__)
-    __asm __volatile ("" : : "g"(chunk) : "memory");
-    #endif
-  }
-  #elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64
-  for(int tries=0; tries<4; tries++) {   // paranoia: at most 4 tries
-    // a cache line is 64b so we can just as well load all at the same time (?)
-    const uint64x2_t vzero1_lo = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields));        // 2x64 bit is_zero
-    const uint64x2_t vzero1_hi = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 2));    // 2x64 bit is_zero
-    const uint64x2_t vzero2_lo = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 4));    // 2x64 bit is_zero
-    const uint64x2_t vzero2_hi = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 6));    // 2x64 bit is_zero
-    const uint32x4_t vzero1    = vuzp1q_u32(vreinterpretq_u32_u64(vzero1_lo),vreinterpretq_u32_u64(vzero1_hi)); // unzip even elements: narrow to 4x32 bit is_zero ()
-    const uint32x4_t vzero2    = vuzp1q_u32(vreinterpretq_u32_u64(vzero2_lo),vreinterpretq_u32_u64(vzero2_hi)); // unzip even elements: narrow to 4x32 bit is_zero ()
-    const uint32x4_t vzero1x   = vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(vzero1), 24));        // shift-right 2x32bit elem by 24: lo 16 bits contain the 2 lo bytes
-    const uint32x4_t vzero2x   = vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(vzero2), 24));
-    const uint16x8_t vzero12   = vreinterpretq_u16_u32(vuzp1q_u32(vzero1x,vzero2x));                           // unzip even 32-bit elements into one vector
-    const uint8x8_t  vzero     = vmovn_u16(vzero12);                                                           // narrow the bottom 16-bits
-    const uint64_t mask = ~vget_lane_u64(vreinterpret_u64_u8(vzero), 0);  // 1 byte for each bfield (0xFF => bfield has a bit set)
-    if (mask==0) return false;
-    mi_assert_internal((mi_ctz(mask)%8) == 0); // tzcnt == 0, 8, 16, 24 , ..
-    const size_t chunk_idx = mi_ctz(mask) / 8;
-    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;
-    // try again
-    // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded
-    // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)
-    #if defined(__GNUC__)
-    __asm __volatile ("" : : "g"(chunk) : "memory");
-    #endif
-  }
-  #else
-  for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
-    if (mi_bchunk_try_find_and_clear_at(chunk, i, pidx)) return true;
-  }
-  #endif
-  return false;  
-}
-
-static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
-  mi_assert_internal(n==1); MI_UNUSED(n);
-  return mi_bchunk_try_find_and_clear(chunk, pidx);
-}
-
-mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {
-  const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[chunk_idx]);
-  // has_set8 has low bit in each byte set if the byte in x == 0xFF
-  const mi_bfield_t has_set8 =
-    ((~b - MI_BFIELD_LO_BIT8) &      // high bit set if byte in x is 0xFF or < 0x7F
-     (b  & MI_BFIELD_HI_BIT8))       // high bit set if byte in x is >= 0x80
-     >> 7;                           // shift high bit to low bit
-  size_t idx;
-  if (mi_bfield_find_least_bit(has_set8, &idx)) { // find least 1-bit
-    mi_assert_internal(idx <= (MI_BFIELD_BITS - 8));
-    mi_assert_internal((idx%8)==0);
-    if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], (mi_bfield_t)0xFF << idx, b, NULL)) {  // unset the byte atomically
-      *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;
-      mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);
-      return true;
-    }
-  }
-  return false;
-}
-
-// find least aligned byte in a chunk with all bits set, and try unset it atomically
-// set `*pidx` to its bit index (0 <= *pidx < MI_BCHUNK_BITS) on success.
-// Used to find medium size pages in the free blocks.
-// todo: try neon version
-static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx) {
-  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
-  while (true) {
-    // since a cache-line is 64b, load all at once
-    const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);
-    const __m256i vec2 = _mm256_load_si256((const __m256i*)chunk->bfields+1);
-    const __m256i cmpv = mi_mm256_ones();
-    const __m256i vcmp1 = _mm256_cmpeq_epi8(vec1, cmpv); // (byte == ~0 ? 0xFF : 0)
-    const __m256i vcmp2 = _mm256_cmpeq_epi8(vec2, cmpv); // (byte == ~0 ? 0xFF : 0)
-    const uint32_t mask1 = _mm256_movemask_epi8(vcmp1);    // mask of most significant bit of each byte
-    const uint32_t mask2 = _mm256_movemask_epi8(vcmp2);    // mask of most significant bit of each byte
-    const uint64_t mask = ((uint64_t)mask2 << 32) | mask1;
-    // mask is inverted, so each bit is 0xFF iff the corresponding byte has a bit set (and thus can be cleared)
-    if (mask==0) return false;
-    const size_t bidx = _tzcnt_u64(mask);          // byte-idx of the byte in the chunk
-    const size_t chunk_idx = bidx / 8;
-    const size_t idx = (bidx % 8)*8;
-    mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);
-    if mi_likely(mi_bfield_atomic_try_clear8(&chunk->bfields[chunk_idx], idx, NULL)) {  // clear it atomically
-      *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;
-      mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);
-      return true;
-    }
-    // try again
-    // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded  }
-  }
-  #else
-    for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
-      if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx)) return true;
-    }
-    return false;
-  #endif
-}
-
-static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
-  mi_assert_internal(n==8); MI_UNUSED(n);
-  return mi_bchunk_try_find_and_clear8(chunk, pidx);
-}
-
-
-// find a sequence of `n` bits in a chunk with `0 < n <= MI_BFIELD_BITS` with all bits set,
-// and try to clear them atomically.
-// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.
-// will cross bfield boundaries.
-mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
-  if (n == 0 || n > MI_BFIELD_BITS) return false;
-  const mi_bfield_t mask = mi_bfield_mask(n, 0);
-  // for all fields in the chunk
-  for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
-    mi_bfield_t b0 = mi_atomic_load_relaxed(&chunk->bfields[i]);
-    mi_bfield_t b = b0;
-    size_t idx;
-
-    // is there a range inside the field?
-    while (mi_bfield_find_least_bit(b, &idx)) { // find least 1-bit
-      if (idx + n > MI_BFIELD_BITS) break; // too short: maybe cross over, or continue with the next field
-
-      const size_t bmask = mask<<idx;
-      mi_assert_internal(bmask>>idx == mask);
-      if ((b&bmask) == bmask) { // found a match with all bits set, try clearing atomically
-        if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[i], bmask, b0, NULL)) {
-          *pidx = (i*MI_BFIELD_BITS) + idx;
-          mi_assert_internal(*pidx < MI_BCHUNK_BITS);
-          mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);
-          return true;
-        }
-        else {
-          // if we failed to atomically commit, reload b and try again from the start
-          b = b0 = mi_atomic_load_acquire(&chunk->bfields[i]);
-        }
-      }
-      else {
-        // advance by clearing the least run of ones, for example, with n>=4, idx=2:
-        // b             = 1111 1101 1010 1100
-        // .. + (1<<idx) = 1111 1101 1011 0000
-        // .. & b        = 1111 1101 1010 0000
-        b = b & (b + (mi_bfield_one() << idx));
-      }
-    }
-
-    // check if we can cross into the next bfield
-    if (b!=0 && i < MI_BCHUNK_FIELDS-1) {
-      const size_t post = mi_bfield_clz(~b);
-      if (post > 0) {
-        const size_t pre = mi_bfield_ctz(~mi_atomic_load_relaxed(&chunk->bfields[i+1]));
-        if (post + pre >= n) {
-          // it fits -- try to claim it atomically
-          const size_t cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - post);
-          if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL)) {
-            // we cleared all atomically
-            *pidx = cidx;
-            mi_assert_internal(*pidx < MI_BCHUNK_BITS);
-            mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);
-            return true;
-          }
-        }
-      }
-    }
-  }
-  return false;
-}
-
-// find a sequence of `n` bits in a chunk with `n <= MI_BCHUNK_BITS` with all bits set,
-// and try to clear them atomically.
-// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.
-// This can cross bfield boundaries.
-static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
-  if (n == 0 || n > MI_BCHUNK_BITS) return false;  // cannot be more than a chunk
-
-  // we first scan ahead to see if there is a range of `n` set bits, and only then try to clear atomically
-  mi_assert_internal(n>0);
-  const size_t skip_count = (n-1)/MI_BFIELD_BITS;
-  size_t cidx;
-  for (size_t i = 0; i < MI_BCHUNK_FIELDS - skip_count; i++)
-  {
-    size_t m = n;   // bits to go
-
-    // first field
-    mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
-    size_t ones = mi_bfield_clz(~b);
-
-    cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - ones);  // start index
-    if (ones >= m) {
-      // we found enough bits already!
-      m = 0;
-    }
-    else if (ones > 0) {
-      // keep scanning further fields until we have enough bits
-      m -= ones;
-      size_t j = 1;   // field count from i
-      while (i+j < MI_BCHUNK_FIELDS) {
-        mi_assert_internal(m > 0);
-        b = mi_atomic_load_relaxed(&chunk->bfields[i+j]);
-        ones = mi_bfield_ctz(~b);
-        if (ones >= m) {
-          // we found enough bits
-          m = 0;
-          break;
-        }
-        else if (ones == MI_BFIELD_BITS) {
-          // not enough yet, proceed to the next field
-          j++;
-          m -= MI_BFIELD_BITS;
-        }
-        else {
-          // the range was not enough, start from scratch
-          i = i + j - 1;  // no need to re-scan previous fields, except the last one (with clz this time)
-          mi_assert_internal(m>0);
-          break;
-        }
-      }
-    }
-
-    // did we find a range?
-    if (m==0) {
-      if (mi_bchunk_try_clearN(chunk, cidx, n, NULL)) {
-        // we cleared all atomically
-        *pidx = cidx;
-        mi_assert_internal(*pidx < MI_BCHUNK_BITS);
-        mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);
-        return true;
-      }
-      // note: if we fail for a small `n` on the first field, we don't rescan that field (as `i` is incremented)
-    }
-    // otherwise continue searching
-  }
-  return false;
-}
-
-
-
-// ------- mi_bchunk_clear_once_set ---------------------------------------
-
-static inline void mi_bchunk_clear_once_set(mi_bchunk_t* chunk, size_t cidx) {
-  mi_assert_internal(cidx < MI_BCHUNK_BITS);
-  const size_t i = cidx / MI_BFIELD_BITS;
-  const size_t idx = cidx % MI_BFIELD_BITS;
-  mi_bfield_atomic_clear_once_set(&chunk->bfields[i], idx);
-}
-
-
-// ------- mi_bitmap_all_are_clear ---------------------------------------
-
-
-// are all bits in a bitmap chunk clear?
-static inline bool mi_bchunk_all_are_clear_relaxed(mi_bchunk_t* chunk) {
-  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
-  const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
-  return mi_mm256_is_zero(vec);
-  #elif MI_OPT_SIMD &&  defined(__AVX2__) && (MI_BCHUNK_BITS==512)
-  // a 64b cache-line contains the entire chunk anyway so load both at once
-  const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);
-  const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);
-  return (mi_mm256_is_zero(_mm256_or_si256(vec1,vec2)));
-  #elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64
-  const uint64x2_t v0 = vld1q_u64((uint64_t*)chunk->bfields);
-  const uint64x2_t v1 = vld1q_u64((uint64_t*)chunk->bfields + 2);
-  const uint64x2_t v2 = vld1q_u64((uint64_t*)chunk->bfields + 4);
-  const uint64x2_t v3 = vld1q_u64((uint64_t*)chunk->bfields + 6);
-  const uint64x2_t v  = vorrq_u64(vorrq_u64(v0,v1),vorrq_u64(v2,v3));
-  return (vmaxvq_u32(vreinterpretq_u32_u64(v)) == 0);
-  #else
-  for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
-    if (mi_atomic_load_relaxed(&chunk->bfields[i]) != 0) return false;
-  }
-  return true;
-  #endif
-}
-
-// are all bits in a bitmap chunk set?
-static inline bool mi_bchunk_all_are_set_relaxed(mi_bchunk_t* chunk) {
-#if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
-  const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
-  return mi_mm256_is_ones(vec);
-#elif MI_OPT_SIMD &&  defined(__AVX2__) && (MI_BCHUNK_BITS==512)
-  // a 64b cache-line contains the entire chunk anyway so load both at once
-  const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);
-  const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);
-  return (mi_mm256_is_ones(_mm256_and_si256(vec1, vec2)));
-#elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64
-  const uint64x2_t v0 = vld1q_u64((uint64_t*)chunk->bfields);
-  const uint64x2_t v1 = vld1q_u64((uint64_t*)chunk->bfields + 2);
-  const uint64x2_t v2 = vld1q_u64((uint64_t*)chunk->bfields + 4);
-  const uint64x2_t v3 = vld1q_u64((uint64_t*)chunk->bfields + 6);
-  const uint64x2_t v  = vandq_u64(vandq_u64(v0,v1),vandq_u64(v2,v3));
-  return (vminvq_u32(vreinterpretq_u32_u64(v)) == 0xFFFFFFFFUL);
-#else
-  for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
-    if (~mi_atomic_load_relaxed(&chunk->bfields[i]) != 0) return false;
-  }
-  return true;
-#endif
-}
-
-
-static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) {
-  for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) {
-    i--;
-    mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
-    size_t idx;
-    if (mi_bsr(b, &idx)) {
-      *pidx = (i*MI_BFIELD_BITS) + idx;
-      return true;
-    }
-  }
-  return false;
-}
-
-static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) {
-  for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) {
-    i--;
-    mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
-    size_t idx;
-    if (mi_bsr(~b, &idx)) {
-      *pidx = (i*MI_BFIELD_BITS) + idx;
-      return true;
-    }
-  }
-  return false;
-}
-
-static size_t mi_bchunk_popcount(mi_bchunk_t* chunk) {
-  size_t popcount = 0;
-  for (size_t i = 0; i < MI_BCHUNK_FIELDS; i++) {
-    const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
-    popcount += mi_bfield_popcount(b);
-  }
-  return popcount;
-}
-
-
-/* --------------------------------------------------------------------------------
- bitmap chunkmap
--------------------------------------------------------------------------------- */
-
-static void mi_bitmap_chunkmap_set(mi_bitmap_t* bitmap, size_t chunk_idx) {
-  mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap));
-  mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL);
-}
-
-static bool mi_bitmap_chunkmap_try_clear(mi_bitmap_t* bitmap, size_t chunk_idx) {
-  mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap));
-  // check if the corresponding chunk is all clear
-  if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) return false;
-  // clear the chunkmap bit
-  mi_bchunk_clear(&bitmap->chunkmap, chunk_idx, NULL);
-  // .. but a concurrent set may have happened in between our all-clear test and the clearing of the
-  // bit in the mask. We check again to catch this situation.
-  if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) {
-    mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL);
-    return false;
-  }
-  return true;
-}
-
-
-/* --------------------------------------------------------------------------------
-  bitmap
--------------------------------------------------------------------------------- */
-
-size_t mi_bitmap_size(size_t bit_count, size_t* pchunk_count) {
-  mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0);
-  bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS);
-  mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT);
-  mi_assert_internal(bit_count > 0);
-  const size_t chunk_count = bit_count / MI_BCHUNK_BITS;
-  mi_assert_internal(chunk_count >= 1);
-  const size_t size = offsetof(mi_bitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE);
-  mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 );
-  if (pchunk_count != NULL) { *pchunk_count = chunk_count;  }
-  return size;
-}
-
-
-// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true
-// returns the size of the bitmap
-size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero) {
-  size_t chunk_count;
-  const size_t size = mi_bitmap_size(bit_count, &chunk_count);
-  if (!already_zero) {
-    _mi_memzero_aligned(bitmap, size);
-  }
-  mi_atomic_store_release(&bitmap->chunk_count, chunk_count);
-  mi_assert_internal(mi_atomic_load_relaxed(&bitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT);
-  return size;
-}
-
-
-// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread.
-static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, size_t idx, size_t n) {
-  mi_assert_internal(n>0);
-
-  // start chunk and index
-  size_t chunk_idx = idx / MI_BCHUNK_BITS;
-  const size_t cidx = idx % MI_BCHUNK_BITS;
-  const size_t ccount = _mi_divide_up(n, MI_BCHUNK_BITS);
-
-  // first update the chunkmap
-  mi_bchunk_setN(cmap, chunk_idx, ccount, NULL);
-
-  // first chunk
-  size_t m = MI_BCHUNK_BITS - cidx;
-  if (m > n) { m = n; }
-  mi_bchunk_setN(&chunks[chunk_idx], cidx, m, NULL);
-
-  // n can be large so use memset for efficiency for all in-between chunks
-  chunk_idx++;
-  n -= m;
-  const size_t mid_chunks = n / MI_BCHUNK_BITS;
-  if (mid_chunks > 0) {
-    _mi_memset(&chunks[chunk_idx], ~0, mid_chunks * MI_BCHUNK_SIZE);
-    chunk_idx += mid_chunks;
-    n -= (mid_chunks * MI_BCHUNK_BITS);
-  }
-
-  // last chunk
-  if (n > 0) {
-    mi_assert_internal(n < MI_BCHUNK_BITS);
-    mi_bchunk_setN(&chunks[chunk_idx], 0, n, NULL);
-  }
-}
-
-// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread.
-void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
-  mi_assert_internal(n>0);
-  mi_assert_internal(idx + n <= mi_bitmap_max_bits(bitmap));
-  mi_bchunks_unsafe_setN(&bitmap->chunks[0], &bitmap->chunkmap, idx, n);
-}
-
-
-
-
-// ------- mi_bitmap_xset ---------------------------------------
-
-// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's).
-bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* palready_set) {
-  mi_assert_internal(n>0);
-  const size_t maxbits = mi_bitmap_max_bits(bitmap);
-  mi_assert_internal(idx + n <= maxbits);
-  if (idx+n > maxbits) { // paranoia
-    if (idx >= maxbits) return false;
-    n = maxbits - idx;
-  }
-
-  // iterate through the chunks
-  size_t chunk_idx = idx / MI_BCHUNK_BITS;
-  size_t cidx = idx % MI_BCHUNK_BITS;
-  bool were_allclear = true;
-  size_t already_set = 0;
-  while (n > 0) {
-    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
-    size_t _already_set = 0;
-    were_allclear = mi_bchunk_setN(&bitmap->chunks[chunk_idx], cidx, m, &_already_set) && were_allclear;
-    already_set += _already_set;
-    mi_bitmap_chunkmap_set(bitmap, chunk_idx); // set afterwards
-    mi_assert_internal(m <= n);
-    n -= m;
-    cidx = 0;
-    chunk_idx++;
-  }
-  if (palready_set != NULL) { *palready_set = already_set;  }
-  return were_allclear;
-}
-
-// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 1's to 0's.
-bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
-  mi_assert_internal(n>0);
-  const size_t maxbits = mi_bitmap_max_bits(bitmap);
-  mi_assert_internal(idx + n <= maxbits);
-  if (idx+n > maxbits) { // paranoia
-    if (idx >= maxbits) return false;
-    n = maxbits - idx;
-  }
-
-  // iterate through the chunks
-  size_t chunk_idx = idx / MI_BCHUNK_BITS;
-  size_t cidx = idx % MI_BCHUNK_BITS;
-  bool were_allset = true;
-  while (n > 0) {
-    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
-    bool maybe_all_clear = false;
-    were_allset = mi_bchunk_clearN(&bitmap->chunks[chunk_idx], cidx, m, &maybe_all_clear) && were_allset;
-    if (maybe_all_clear) { mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); }
-    mi_assert_internal(m <= n);
-    n -= m;
-    cidx = 0;
-    chunk_idx++;
-  }
-  return were_allset;
-}
-
-// Count bits set in a range of `n` bits.
-size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n) {
-  mi_assert_internal(n>0);
-  const size_t maxbits = mi_bitmap_max_bits(bitmap);
-  mi_assert_internal(idx + n <= maxbits);
-  if (idx+n > maxbits) { // paranoia
-    if (idx >= maxbits) return 0;
-    n = maxbits - idx;
-  }
-
-  // iterate through the chunks
-  size_t chunk_idx = idx / MI_BCHUNK_BITS;
-  size_t cidx = idx % MI_BCHUNK_BITS;
-  size_t popcount = 0;
-  while (n > 0) {
-    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
-    popcount += mi_bchunk_popcountN(&bitmap->chunks[chunk_idx], cidx, m);
-    mi_assert_internal(m <= n);
-    n -= m;
-    cidx = 0;
-    chunk_idx++;
-  }
-  return popcount;
-}
-
-
-// Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0)
-bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) {
-  return mi_bitmap_setN(bitmap, idx, 1, NULL);
-}
-
-bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx) {
-  return mi_bitmap_clearN(bitmap, idx, 1);
-}
-
-
-
-// ------- mi_bitmap_is_xset ---------------------------------------
-
-// Is a sequence of n bits already all set/cleared?
-bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n) {
-  mi_assert_internal(n>0);
-  const size_t maxbits = mi_bitmap_max_bits(bitmap);
-  mi_assert_internal(idx + n <= maxbits);
-  if (idx+n > maxbits) { // paranoia
-    if (idx >= maxbits) return false;
-    n = maxbits - idx;
-  }
-
-  // iterate through the chunks
-  size_t chunk_idx = idx / MI_BCHUNK_BITS;
-  size_t cidx = idx % MI_BCHUNK_BITS;
-  bool xset = true;
-  while (n > 0 && xset) {
-    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
-    xset = mi_bchunk_is_xsetN(set, &bitmap->chunks[chunk_idx], cidx, m) && xset;
-    mi_assert_internal(m <= n);
-    n -= m;
-    cidx = 0;
-    chunk_idx++;
-  }
-  return xset;
-}
-
-bool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap) {
-  return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, 0, mi_bitmap_max_bits(bitmap));
-}
-
-/* --------------------------------------------------------------------------------
-  Iterate through a bfield
--------------------------------------------------------------------------------- */
-
-// Cycle iteration through a bitfield. This is used to space out threads
-// so there is less chance of contention. When searching for a free page we
-// like to first search only the accessed part (so we reuse better). This
-// high point is called the `cycle`.
-//
-// We then iterate through the bitfield as:
-// first: [start, cycle>
-// then : [0, start>
-// then : [cycle, MI_BFIELD_BITS>
-//
-// The start is determined usually as `tseq % cycle` to have each thread
-// start at a different spot.
-// - We use `popcount` to improve branch prediction (maybe not needed? can we simplify?)
-// - The `cycle_mask` is the part `[start, cycle>`.
-#define mi_bfield_iterate(bfield,start,cycle,name_idx,SUF) { \
-  mi_assert_internal(start <= cycle); \
-  mi_assert_internal(start < MI_BFIELD_BITS); \
-  mi_assert_internal(cycle <= MI_BFIELD_BITS); \
-  const mi_bfield_t _cycle_mask##SUF = mi_bfield_mask(cycle - start, start); \
-  size_t _bcount##SUF = mi_bfield_popcount(bfield); \
-  mi_bfield_t _b##SUF = bfield & _cycle_mask##SUF; /* process [start, cycle> first*/\
-  while(_bcount##SUF > 0) { \
-    _bcount##SUF--;\
-    if (_b##SUF==0) { _b##SUF = bfield & ~_cycle_mask##SUF; } /* process [0,start> + [cycle, MI_BFIELD_BITS> next */ \
-    /* size_t name_idx; */ \
-    const bool _found##SUF = mi_bfield_find_least_bit(_b##SUF,&name_idx); \
-    _b##SUF = mi_bfield_clear_least_bit(_b##SUF); /* clear early so `continue` works */ \
-    mi_assert_internal(_found##SUF); MI_UNUSED(_found##SUF); \
-    { \
-
-#define mi_bfield_iterate_end(SUF) \
-    } \
-  } \
-}
-
-
-#define mi_bfield_cycle_iterate(bfield,tseq,cycle,name_idx,SUF) { \
-  const size_t _start##SUF = (uint32_t)(tseq) % (uint32_t)(cycle); /* or: 0 to always search from the start? */\
-  mi_bfield_iterate(bfield,_start##SUF,cycle,name_idx,SUF)
-
-#define mi_bfield_cycle_iterate_end(SUF) \
-  mi_bfield_iterate_end(SUF); \
-}
-
-
-/* --------------------------------------------------------------------------------
-  mi_bitmap_find
-  (used to find free pages)
--------------------------------------------------------------------------------- */
-
-typedef bool (mi_bitmap_visit_fun_t)(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* idx, void* arg1, void* arg2);
-
-// Go through the bitmap and for every sequence of `n` set bits, call the visitor function.
-// If it returns `true` stop the search.
-static inline bool mi_bitmap_find(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx, mi_bitmap_visit_fun_t* on_find, void* arg1, void* arg2)
-{
-  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
-  for (size_t i = 0; i < chunkmap_max; i++) {
-    // and for each chunkmap entry we iterate over its bits to find the chunks
-    const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
-    size_t hi;
-    if (mi_bfield_find_highest_bit(cmap_entry, &hi)) {
-      size_t eidx = 0;
-      mi_bfield_cycle_iterate(cmap_entry, tseq%8, hi+1, eidx, Y) // reduce the tseq to 8 bins to reduce using extra memory (see `mstress`)
-      {
-        mi_assert_internal(eidx <= MI_BFIELD_BITS);
-        const size_t chunk_idx = i*MI_BFIELD_BITS + eidx;
-        mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));
-        if ((*on_find)(bitmap, chunk_idx, n, pidx, arg1, arg2)) {
-          return true;
-        }
-      }
-      mi_bfield_cycle_iterate_end(Y);
-    }
-  }
-  return false;
-}
-
-
-/* --------------------------------------------------------------------------------
-  Bitmap: try_find_and_claim  -- used to allocate abandoned pages
-  note: the compiler will fully inline the indirect function call
--------------------------------------------------------------------------------- */
-
-typedef struct mi_claim_fun_data_s {
-  mi_arena_t*   arena;  
-} mi_claim_fun_data_t;
-
-static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* pidx, void* arg1, void* arg2)
-{
-  mi_assert_internal(n==1); MI_UNUSED(n);
-  mi_claim_fun_t* claim_fun = (mi_claim_fun_t*)arg1;
-  mi_claim_fun_data_t* claim_data = (mi_claim_fun_data_t*)arg2;
-  size_t cidx;
-  if mi_likely(mi_bchunk_try_find_and_clear(&bitmap->chunks[chunk_idx], &cidx)) {
-    const size_t slice_index = (chunk_idx * MI_BCHUNK_BITS) + cidx;
-    mi_assert_internal(slice_index < mi_bitmap_max_bits(bitmap));
-    bool keep_set = true;
-    if ((*claim_fun)(slice_index, claim_data->arena, &keep_set)) {
-      // success!
-      mi_assert_internal(!keep_set);
-      *pidx = slice_index;
-      return true;
-    }
-    else {
-      // failed to claim it, set abandoned mapping again (unless the page was freed)
-      if (keep_set) {
-        const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL);
-        mi_assert_internal(wasclear); MI_UNUSED(wasclear);
-      }
-    }
-  }
-  else {
-    // we may find that all are cleared only on a second iteration but that is ok as
-    // the chunkmap is a conservative approximation.
-    mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx);
-  }
-  return false;
-}
-
-// Find a set bit in the bitmap and try to atomically clear it and claim it.
-// (Used to find pages in the pages_abandoned bitmaps.)
-mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,
-  mi_claim_fun_t* claim, mi_arena_t* arena )
-{
-  mi_claim_fun_data_t claim_data = { arena };
-  return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, &claim_data);
-}
-
-
-bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx) {
-  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
-  for (size_t i = chunkmap_max; i > 0; ) {
-    i--;
-    mi_bfield_t cmap = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
-    size_t cmap_idx;
-    if (mi_bsr(cmap,&cmap_idx)) {
-      // highest chunk
-      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
-      size_t cidx;
-      if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) {
-        *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
-        return true;
-      }
-    }
-  }
-  return false;
-}
-
-// Return count of all set bits in a bitmap.
-size_t mi_bitmap_popcount(mi_bitmap_t* bitmap) {
-  // for all chunkmap entries
-  size_t popcount = 0;
-  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
-  for (size_t i = 0; i < chunkmap_max; i++) {
-    mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
-    size_t cmap_idx;
-    // for each chunk (corresponding to a set bit in a chunkmap entry)
-    while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {
-      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
-      // count bits in a chunk
-      popcount += mi_bchunk_popcount(&bitmap->chunks[chunk_idx]);
-    }
-  }
-  return popcount;
-}
-
-
-
-// Clear a bit once it is set.
-void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx) {
-  mi_assert_internal(idx < mi_bitmap_max_bits(bitmap));
-  const size_t chunk_idx = idx / MI_BCHUNK_BITS;
-  const size_t cidx = idx % MI_BCHUNK_BITS;
-  mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));
-  mi_bchunk_clear_once_set(&bitmap->chunks[chunk_idx], cidx);
-}
-
-
-// Visit all set bits in a bitmap.
-// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore?
-bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) {
-  // for all chunkmap entries
-  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
-  for(size_t i = 0; i < chunkmap_max; i++) {
-    mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
-    size_t cmap_idx;
-    // for each chunk (corresponding to a set bit in a chunkmap entry)
-    while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {
-      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
-      // for each chunk field
-      mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];
-      for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
-        const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
-        mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[j]);
-        size_t bidx;
-        while (mi_bfield_foreach_bit(&b, &bidx)) {
-          const size_t idx = base_idx + bidx;
-          if (!visit(idx, 1, arena, arg)) return false;
-        }
-      }
-    }
-  }
-  return true;
-}
-
-// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible.
-// Also clear those ranges atomically.
-// Used by purging to purge larger ranges when possible
-// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore?
-bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) {
-  // for all chunkmap entries
-  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
-  for (size_t i = 0; i < chunkmap_max; i++) {
-    mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
-    size_t cmap_idx;
-    // for each chunk (corresponding to a set bit in a chunkmap entry)
-    while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {
-      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
-      // for each chunk field
-      mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];
-      for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
-        const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
-        mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0);
-        #if MI_DEBUG > 1
-        const size_t bpopcount = mi_popcount(b);
-        size_t rngcount = 0;
-        #endif
-        size_t bidx;
-        while (mi_bfield_find_least_bit(b, &bidx)) {
-          size_t rng = mi_ctz(~(b>>bidx)); // all the set bits from bidx
-          #if MI_DEBUG > 1
-          rngcount += rng;
-          #endif
-          const size_t idx = base_idx + bidx;
-          mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS);
-          mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS);
-          mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap));
-          if (!visit(idx, rng, arena, arg)) {
-            // break early: reset the non-visited bits
-            if (b!=0) {
-              mi_atomic_or_relaxed(&chunk->bfields[j], b);
-            }
-            return false;
-          }
-          // clear rng bits in b
-          b = b & ~mi_bfield_mask(rng, bidx);
-        }
-        mi_assert_internal(rngcount == bpopcount);
-      }
-    }
-  }
-  return true;
-}
-
-// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible,
-// but only in chunks of at least `rngslices` slices (that are also aligned at `rngslices`)
-// and clear those ranges atomically.
-// However, the `rngslices` are capped at `MI_BFIELD_BITS` at most.
-// Used by purging to purge larger ranges when possible. With transparent huge pages we only
-// want to purge whole huge pages (2 MiB) at a time which is what the `rngslices` parameter achieves.
-bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) 
-{
-  // use the generic routine for `rngslices<=1` (as that one finds longest ranges at a time)
-  if (rngslices<=1) {
-    return _mi_bitmap_forall_setc_ranges(bitmap, visit, arena, arg);
-  }
-  // mi_assert_internal(rngslices <= MI_BFIELD_BITS);  
-  if (rngslices > MI_BFIELD_BITS) { rngslices = MI_BFIELD_BITS;  } // cap at MI_BFIELD_BITS at most
-
-  // for all chunkmap entries
-  const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);
-  for (size_t i = 0; i < chunkmap_max; i++) {
-    mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
-    size_t cmap_idx;
-    // for each chunk (corresponding to a set bit in a chunkmap entry)
-    while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {
-      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
-      // for each chunk field
-      mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];
-      for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
-        const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
-        mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0);   // atomic clear
-        mi_bfield_t skipped = 0;                                                          // but track which bits we skip so we can restore them
-        for(size_t shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) {  // per `rngslices` to keep alignment
-          const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift);
-          if ((b & rngmask) == rngmask) {
-            const size_t idx = base_idx + shift;
-            if (!visit(idx, rngslices, arena, arg)) {
-              // break early: restore non-visited entries
-              mi_bfield_t notyet_visited = 0;
-              if (shift + rngslices < MI_BFIELD_BITS) {
-                notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices)));
-              }
-              mi_assert_internal((notyet_visited & skipped) == 0);
-              if ((notyet_visited | skipped) != 0) {
-                mi_atomic_or_relaxed(&chunk->bfields[j], notyet_visited | skipped);
-              }
-              return false;
-            }
-          }
-          else {
-            skipped = skipped | (b & rngmask);
-          }          
-        } 
-        
-        if (skipped != 0) {
-          mi_atomic_or_relaxed(&chunk->bfields[j], skipped);
-        }
-      }
-    }
-  }
-  return true;
-}
-
-
-/* --------------------------------------------------------------------------------
-  binned bitmap's
--------------------------------------------------------------------------------- */
-
-
-size_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) {
-  // mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0);
-  bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS);
-  mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT);
-  mi_assert_internal(bit_count > 0);
-  const size_t chunk_count = bit_count / MI_BCHUNK_BITS;
-  mi_assert_internal(chunk_count >= 1);
-  const size_t size = offsetof(mi_bbitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE);
-  mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 );
-  if (pchunk_count != NULL) { *pchunk_count = chunk_count;  }
-  return size;
-}
-
-// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true
-// returns the size of the bitmap
-size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) {
-  size_t chunk_count;
-  const size_t size = mi_bbitmap_size(bit_count, &chunk_count);
-  if (!already_zero) {
-    _mi_memzero_aligned(bbitmap, size);
-  }
-  mi_atomic_store_release(&bbitmap->chunk_count, chunk_count);
-  mi_assert_internal(mi_atomic_load_relaxed(&bbitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT);
-  return size;
-}
-
-void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
-  mi_assert_internal(n>0);
-  mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap));
-  mi_bchunks_unsafe_setN(&bbitmap->chunks[0], &bbitmap->chunkmap, idx, n);
-}
-
-bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) {
-  const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap);
-  const size_t chunkmap_max = _mi_divide_up(chunk_count, MI_BFIELD_BITS);
-  size_t skip_at_top = chunk_count % MI_BFIELD_BITS;
-  for (size_t i = chunkmap_max; i > 0; ) {
-    i--;
-    mi_bfield_t cmap = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[i]);
-    size_t cmap_idx;
-    // don't consider top 0 bits; set those to 1 here
-    if (skip_at_top > 0) {
-      const size_t mask_top = (~mi_bfield_zero()) << (MI_BFIELD_BITS - skip_at_top);
-      skip_at_top = 0;   // only for the first iteration
-      cmap |= mask_top;
-    }
-    if (mi_bsr(~cmap, &cmap_idx)) {
-      // highest chunk
-      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
-      size_t cidx;
-      if (mi_bchunk_bsr_inv(&bbitmap->chunks[chunk_idx], &cidx)) {
-        *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
-        return true;
-      }
-    }
-  }
-  return false;
-}
-
-
-/* --------------------------------------------------------------------------------
- binned bitmap used to track free slices
--------------------------------------------------------------------------------- */
-
-// Assign a specific size bin to a chunk
-static void mi_bbitmap_set_chunk_bin(mi_bbitmap_t* bbitmap, size_t chunk_idx, mi_chunkbin_t bin) {
-  mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap));
-  for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {
-    if (ibin == bin) {
-      const bool was_clear = mi_bchunk_set(& bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);
-      if (was_clear) { mi_os_stat_increase(chunk_bins[ibin],1); }
-    }
-    else {
-      const bool was_set = mi_bchunk_clear(&bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);
-      if (was_set) { mi_os_stat_decrease(chunk_bins[ibin],1); }
-    }
-  }
-}
-
-mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunkmap_t* chunkmap_bins, size_t chunk_idx) {
-  for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {
-    if (mi_bchunk_is_xsetN(MI_BIT_SET, &chunkmap_bins[ibin], chunk_idx, 1)) {
-      return ibin;
-    }
-  }
-  return MI_CBIN_NONE;
-}
-
-// Track the index of the highest chunk that is accessed.
-static void mi_bbitmap_chunkmap_set_max(mi_bbitmap_t* bbitmap, size_t chunk_idx) {
-  size_t oldmax = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed);
-  if mi_unlikely(chunk_idx > oldmax) {
-    mi_atomic_cas_strong_relaxed(&bbitmap->chunk_max_accessed, &oldmax, chunk_idx);
-  }
-}
-
-// Set a bit in the chunkmap
-static void mi_bbitmap_chunkmap_set(mi_bbitmap_t* bbitmap, size_t chunk_idx, bool check_all_set) {
-  mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap));
-  if (check_all_set) {
-    if (mi_bchunk_all_are_set_relaxed(&bbitmap->chunks[chunk_idx])) {
-      // all slices are free in this chunk: return back to the NONE bin
-      mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, MI_CBIN_NONE);
-    }
-  }
-  mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL);
-  mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx);
-}
-
-static bool mi_bbitmap_chunkmap_try_clear(mi_bbitmap_t* bbitmap, size_t chunk_idx) {
-  mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap));
-  // check if the corresponding chunk is all clear
-  if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) return false;
-  // clear the chunkmap bit
-  mi_bchunk_clear(&bbitmap->chunkmap, chunk_idx, NULL);
-  // .. but a concurrent set may have happened in between our all-clear test and the clearing of the
-  // bit in the mask. We check again to catch this situation. (note: mi_bchunk_clear must be acq-rel)
-  if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) {
-    mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL);
-    return false;
-  }
-  mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx);
-  return true;
-}
-
-
-/* --------------------------------------------------------------------------------
-  mi_bbitmap_setN, try_clearN, and is_xsetN
-  (used to find free pages)
--------------------------------------------------------------------------------- */
-
-// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's).
-bool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
-  mi_assert_internal(n>0);
-  const size_t maxbits = mi_bbitmap_max_bits(bbitmap);
-  mi_assert_internal(idx + n <= maxbits);
-  if (idx+n > maxbits) { // paranoia
-    if (idx >= maxbits) return false;
-    n = maxbits - idx;
-  }
-
-  // iterate through the chunks
-  size_t chunk_idx = idx / MI_BCHUNK_BITS;
-  size_t cidx = idx % MI_BCHUNK_BITS;
-  bool were_allclear = true;
-  while (n > 0) {
-    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
-    were_allclear = mi_bchunk_setN(&bbitmap->chunks[chunk_idx], cidx, m, NULL) && were_allclear;
-    mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, true); // set afterwards
-    mi_assert_internal(m <= n);
-    n -= m;
-    cidx = 0;
-    chunk_idx++;
-  }
-  return were_allclear;
-}
-
-// ------- mi_bbitmap_try_clearNC ---------------------------------------
-
-// Try to clear `n` bits at `idx` where `n <= MI_BCHUNK_BITS`.
-bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
-  mi_assert_internal(n>0);
-  mi_assert_internal(n<=MI_BCHUNK_BITS);
-  mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap));
-
-  const size_t chunk_idx = idx / MI_BCHUNK_BITS;
-  const size_t cidx = idx % MI_BCHUNK_BITS;
-  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);  // don't cross chunks (for now)
-  mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap));
-  if (cidx + n > MI_BCHUNK_BITS) return false;
-  bool maybe_all_clear = false;
-  const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear);
-  if (cleared && maybe_all_clear) { mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); }
-  // note: we don't set the size class for an explicit try_clearN (only used by purging)
-  return cleared;
-}
-
-
-
-// ------- mi_bbitmap_is_xset ---------------------------------------
-
-// Is a sequence of n bits already all set/cleared?
-bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
-  mi_assert_internal(n>0);
-  const size_t maxbits = mi_bbitmap_max_bits(bbitmap);
-  mi_assert_internal(idx + n <= maxbits);
-  if (idx+n > maxbits) { // paranoia
-    if (idx >= maxbits) return false;
-    n = maxbits - idx;
-  }
-
-  // iterate through the chunks
-  size_t chunk_idx = idx / MI_BCHUNK_BITS;
-  size_t cidx = idx % MI_BCHUNK_BITS;
-  bool xset = true;
-  while (n > 0 && xset) {
-    const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);
-    xset = mi_bchunk_is_xsetN(set, &bbitmap->chunks[chunk_idx], cidx, m) && xset;
-    mi_assert_internal(m <= n);
-    n -= m;
-    cidx = 0;
-    chunk_idx++;
-  }
-  return xset;
-}
-
-
-
-
-/* --------------------------------------------------------------------------------
-  mi_bbitmap_find
-  (used to find free pages)
--------------------------------------------------------------------------------- */
-
-typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx);
-
-// Go through the bbitmap and for every sequence of `n` set bits, call the visitor function.
-// If it returns `true` stop the search.
-//
-// This is used for finding free blocks and it is important to be efficient (with 2-level bitscan)
-// but also reduce fragmentation (through size bins).
-static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find)
-{
-  // we space out threads to reduce contention
-  const size_t cmap_max_count  = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS);
-  const size_t chunk_acc       = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed);
-  const size_t cmap_acc        = chunk_acc / MI_BFIELD_BITS;
-  const size_t cmap_acc_bits   = 1 + (chunk_acc % MI_BFIELD_BITS);
-
-  // create a mask over the chunkmap entries to iterate over them efficiently
-  mi_assert_internal(MI_BFIELD_BITS >= MI_BCHUNK_FIELDS);
-  const mi_bfield_t cmap_mask  = mi_bfield_mask(cmap_max_count,0);
-  const size_t cmap_cycle      = cmap_acc+1;
-  const mi_chunkbin_t bbin = mi_chunkbin_of(n);
-  // visit each cmap entry
-  size_t cmap_idx = 0;
-  mi_bfield_cycle_iterate(cmap_mask, tseq, cmap_cycle, cmap_idx, X)
-  {
-    // and for each chunkmap entry we iterate over its bits to find the chunks
-    const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[cmap_idx]);
-    const size_t cmap_entry_cycle = (cmap_idx != cmap_acc ? MI_BFIELD_BITS : cmap_acc_bits);
-    if (cmap_entry == 0) {
-      continue;
-    }
-
-    // get size bin masks
-    mi_bfield_t cmap_bins[MI_CBIN_COUNT] = { 0 };
-    cmap_bins[MI_CBIN_NONE] = cmap_entry;
-    for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {
-      const mi_bfield_t cmap_bin = mi_atomic_load_relaxed(&bbitmap->chunkmap_bins[ibin].bfields[cmap_idx]);
-      cmap_bins[ibin] = cmap_bin & cmap_entry;
-      cmap_bins[MI_CBIN_NONE] &= ~cmap_bin;      // clear bits that are in an assigned size bin
-    }
-
-    // consider only chunks for a particular size bin at a time
-    // this picks the best bin only within a cmap entry (~ 1GiB address space), but avoids multiple
-    // iterations through all entries.
-    mi_assert_internal(bbin < MI_CBIN_NONE);
-    for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin <= MI_CBIN_NONE;
-          // skip from bbin to NONE (so, say, a SMALL will never be placed in a OTHER, MEDIUM, or LARGE chunk to reduce fragmentation)
-          ibin = (ibin == bbin ? MI_CBIN_NONE : mi_chunkbin_inc(ibin)))
-    {
-      mi_assert_internal(ibin < MI_CBIN_COUNT);
-      const mi_bfield_t cmap_bin = cmap_bins[ibin];
-      size_t eidx = 0;
-      mi_bfield_cycle_iterate(cmap_bin, tseq, cmap_entry_cycle, eidx, Y)
-      {
-        // assertion doesn't quite hold as the max_accessed may be out-of-date
-        // mi_assert_internal(cmap_entry_cycle > eidx || ibin == MI_CBIN_NONE);
-
-        // get the chunk
-        const size_t chunk_idx = cmap_idx*MI_BFIELD_BITS + eidx;
-        mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx];
-
-        size_t cidx;
-        if ((*on_find)(chunk, n, &cidx)) {
-          if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin
-            // this chunk is now reserved for the `bbin` size class
-            mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin);
-          }
-          *pidx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
-          mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap));
-          return true;
-        }
-        else {
-          // todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?`
-          // we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation.
-          mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);
-        }
-      }
-      mi_bfield_cycle_iterate_end(Y);
-    }
-  }
-  mi_bfield_cycle_iterate_end(X);
-  return false;
-}
-
-/* --------------------------------------------------------------------------------
-  mi_bbitmap_try_find_and_clear -- used to find free pages
-  note: the compiler will fully inline the indirect function calls
--------------------------------------------------------------------------------- */
-
-bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) {
-  return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 1, pidx, &mi_bchunk_try_find_and_clear_1);
-}
-
-bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) {
-  return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 8, pidx, &mi_bchunk_try_find_and_clear_8);
-}
-
-// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) {
-//   return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, MI_BFIELD_BITS, pidx, &mi_bchunk_try_find_and_clear_X);
-// }
-
-bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
-  mi_assert_internal(n<=MI_BFIELD_BITS);
-  return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNX);
-}
-
-bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
-  mi_assert_internal(n<=MI_BCHUNK_BITS);
-  return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNC);
-}
-
-
-/* --------------------------------------------------------------------------------
-  mi_bbitmap_try_find_and_clear for huge objects spanning multiple chunks
--------------------------------------------------------------------------------- */
-
-// Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunks
-static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) {
-  mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap));
-
-  size_t m = n;      // bits to go
-  size_t count = 0;  // chunk count
-  while (m > 0) {
-    mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];
-    if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL)) {
-      goto rollback;
-    }
-    m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS);
-    count++;
-  }
-  return true;
-
-rollback:
-  // we only need to reset chunks the we just fully cleared
-  while (count > 0) {
-    count--;
-    mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];
-    mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL);
-  }
-  return false;
-}
-
-// Go through the bbitmap to find a sequence of `n` bits and clear them atomically where `n > MI_ARENA_MAX_CHUNK_OBJ_SIZE`
-// Since these are very large object allocations we always search from the start and only consider starting at the start
-// of a chunk (for fragmentation and efficiency).
-// Todo: for now we try to find full empty chunks to cover `n` but we can allow a partial chunk at the end
-// Todo: This scans directly through the chunks -- we might want to consult the cmap as well?
-bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
-  MI_UNUSED(tseq);
-  mi_assert(n > 0); if (n==0) { return false; }
-
-  const size_t chunk_max = mi_bbitmap_chunk_count(bbitmap);
-  const size_t chunk_req = _mi_divide_up(n, MI_BCHUNK_BITS);  // minimal number of chunks needed
-  if (chunk_max < chunk_req) { return false; }
-
-  // iterate through the chunks
-  size_t chunk_idx = 0;
-  while (chunk_idx <= chunk_max - chunk_req)
-  {
-    size_t count = 0;  // chunk count
-    do {
-      mi_assert_internal(chunk_idx + count < chunk_max);
-      mi_bchunk_t* const chunk = &bbitmap->chunks[chunk_idx + count];
-      if (!mi_bchunk_all_are_set_relaxed(chunk)) {
-        break;
-      }
-      else {
-        count++;
-      }
-    }
-    while (count < chunk_req);
-
-    // did we find a suitable range?
-    if (count == chunk_req) {
-      // now try to claim it!
-      if (mi_bchunk_try_clearN_(bbitmap, chunk_idx, n)) {
-        *pidx = (chunk_idx * MI_BCHUNK_BITS);
-        for (size_t i = 0; i < count; i++) {
-          mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx + i, MI_CBIN_HUGE);
-        }
-        mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap));
-        return true;
-      }
-    }
-
-    // keep searching but skip the scanned range
-    chunk_idx += count+1;
-  }
-  return false;
-}
-
-
-
-
-
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.h
deleted file mode 100644
index ca51a4e0..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/bitmap.h
+++ /dev/null
@@ -1,343 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* ----------------------------------------------------------------------------
-Concurrent bitmap that can set/reset sequences of bits atomically
----------------------------------------------------------------------------- */
-#pragma once
-#ifndef MI_BITMAP_H
-#define MI_BITMAP_H
-
-/* --------------------------------------------------------------------------------
-  Atomic bitmaps with release/acquire guarantees:
-
-  `mi_bfield_t`: is a single machine word that can efficiently be bit counted (usually `size_t`)
-      each bit usually represents a single MI_ARENA_SLICE_SIZE in an arena (64 KiB).
-      We need 16K bits to represent a 1GiB arena.
-
-  `mi_bchunk_t`: a chunk of bfield's of a total of MI_BCHUNK_BITS (= 512 on 64-bit, 256 on 32-bit)
-      allocations never span across chunks -- so MI_ARENA_MAX_OBJ_SIZE is the number
-      of bits in a chunk times the MI_ARENA_SLICE_SIZE (512 * 64KiB = 32 MiB).
-      These chunks are cache-aligned and we can use AVX2/AVX512/NEON/SVE/SVE2/etc. instructions
-      to scan for bits (perhaps) more efficiently.
-
-      We allocate byte-sized ranges aligned to bytes in the bfield, and bfield-sized
-      ranges aligned to a bfield.
-
-    Searching linearly through the chunks would be too slow (16K bits per GiB).
-    Instead we add a "chunkmap" to do a two-level search (more or less a btree of depth 2).
-
-   `mi_bchunkmap_t` (== `mi_bchunk_t`): for each chunk we track if it has (potentially) any bit set.
-      The chunkmap has 1 bit per chunk that is set if the chunk potentially has a bit set.
-      This is used to avoid scanning every chunk. (and thus strictly an optimization)
-      It is conservative: it is fine to set a bit in the chunk map even if the chunk turns out
-      to have no bits set. It is also allowed to briefly have a clear bit even if the
-      chunk has bits set -- as long as we guarantee that the bit will be set later on;
-      (this allows us to set the chunkmap bit right after we set a bit in the corresponding chunk).
-
-      However, when we clear a bit in a chunk, and the chunk is indeed all clear, we
-      cannot safely clear the bit corresponding to the chunk in the chunkmap since it
-      may race with another thread setting a bit in the same chunk. Therefore, when
-      clearing, we first test if a chunk is clear, then clear the chunkmap bit, and
-      then test again to catch any set bits that we may have missed.
-
-      Since the chunkmap may thus be briefly out-of-sync, this means that we may sometimes
-      not find a free page even though it's there (but we accept this as we avoid taking
-      full locks). (Another way to do this is to use an epoch but we like to avoid that complexity
-      for now).
-
-   `mi_bitmap_t`: a bitmap with N chunks. A bitmap has a chunkmap of MI_BCHUNK_BITS (512)
-      and thus has at most 512 chunks (=2^18 bits x 64 KiB slices = 16 GiB max arena size).
-      The minimum is 1 chunk which is a 32 MiB arena.
-
-   For now, the implementation assumes MI_HAS_FAST_BITSCAN and uses trailing-zero-count
-   and pop-count (but we think it can be adapted work reasonably well on older hardware too)
---------------------------------------------------------------------------------------------- */
-
-// A word-size bit field.
-typedef size_t mi_bfield_t;
-
-#define MI_BFIELD_BITS_SHIFT         (MI_SIZE_SHIFT+3)
-#define MI_BFIELD_BITS               (1 << MI_BFIELD_BITS_SHIFT)
-#define MI_BFIELD_SIZE               (MI_BFIELD_BITS/8)
-#define MI_BFIELD_LO_BIT8            (((~(mi_bfield_t)0))/0xFF)         // 0x01010101 ..
-#define MI_BFIELD_HI_BIT8            (MI_BFIELD_LO_BIT8 << 7)           // 0x80808080 ..
-
-#define MI_BCHUNK_SIZE               (MI_BCHUNK_BITS / 8)
-#define MI_BCHUNK_FIELDS             (MI_BCHUNK_BITS / MI_BFIELD_BITS)  // 8 on both 64- and 32-bit
-
-
-// some compiler (msvc in C mode) cannot have expressions in the alignment attribute
-#if MI_BCHUNK_SIZE==64
-#define mi_decl_bchunk_align  mi_decl_align(64)
-#elif MI_BCHUNK_SIZE==32
-#define mi_decl_bchunk_align  mi_decl_align(32)
-#else
-#define mi_decl_bchunk_align  mi_decl_align(MI_BCHUNK_SIZE)
-#endif
-
-
-// A bitmap chunk contains 512 bits on 64-bit  (256 on 32-bit)
-typedef mi_decl_bchunk_align struct mi_bchunk_s {
-  _Atomic(mi_bfield_t) bfields[MI_BCHUNK_FIELDS];
-} mi_bchunk_t;
-
-
-// The chunkmap has one bit per corresponding chunk that is set if the chunk potentially has bits set.
-// The chunkmap is itself a chunk.
-typedef mi_bchunk_t mi_bchunkmap_t;
-
-#define MI_BCHUNKMAP_BITS             MI_BCHUNK_BITS
-
-#define MI_BITMAP_MAX_CHUNK_COUNT     (MI_BCHUNKMAP_BITS)
-#define MI_BITMAP_MIN_CHUNK_COUNT     (1)
-#if MI_SIZE_BITS > 32
-#define MI_BITMAP_DEFAULT_CHUNK_COUNT     (64)  // 2 GiB on 64-bit -- this is for the page map
-#else
-#define MI_BITMAP_DEFAULT_CHUNK_COUNT      (1)
-#endif
-#define MI_BITMAP_MAX_BIT_COUNT       (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS)  // 16 GiB arena
-#define MI_BITMAP_MIN_BIT_COUNT       (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS)  // 32 MiB arena
-#define MI_BITMAP_DEFAULT_BIT_COUNT   (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS)  // 2 GiB arena
-
-
-// An atomic bitmap
-typedef mi_decl_bchunk_align struct mi_bitmap_s {
-  _Atomic(size_t)  chunk_count;         // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)
-  size_t           _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 1];    // suppress warning on msvc
-  mi_bchunkmap_t   chunkmap;
-  mi_bchunk_t      chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT];        // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT
-} mi_bitmap_t;
-
-
-static inline size_t mi_bitmap_chunk_count(const mi_bitmap_t* bitmap) {
-  return mi_atomic_load_relaxed(&((mi_bitmap_t*)bitmap)->chunk_count);
-}
-
-static inline size_t mi_bitmap_max_bits(const mi_bitmap_t* bitmap) {
-  return (mi_bitmap_chunk_count(bitmap) * MI_BCHUNK_BITS);
-}
-
-
-
-/* --------------------------------------------------------------------------------
-  Atomic bitmap operations
--------------------------------------------------------------------------------- */
-
-// Many operations are generic over setting or clearing the bit sequence: we use `mi_xset_t` for this (true if setting, false if clearing)
-typedef bool  mi_xset_t;
-#define MI_BIT_SET    (true)
-#define MI_BIT_CLEAR  (false)
-
-
-// Required size of a bitmap to represent `bit_count` bits.
-size_t mi_bitmap_size(size_t bit_count, size_t* chunk_count);
-
-// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true
-// returns the size of the bitmap.
-size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero);
-
-// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).
-// Not atomic so only use if still local to a thread.
-void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n);
-
-
-// Set a bit in the bitmap; returns `true` if it atomically transitioned from 0 to 1
-bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx);
-
-// Clear a bit in the bitmap; returns `true` if it atomically transitioned from 1 to 0
-bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx);
-
-// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 0's to 1's
-// If `already_set` is not NULL, it is set to count of bits were already all set.
-// (this is used for correct statistics if commiting over a partially committed area)
-bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_set);
-
-// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 1's to 0's
-bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n);
-
-
-// Is a sequence of n bits already all set/cleared?
-bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n);
-
-// Is the bitmap completely clear?
-bool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap);
-
-// Is a sequence of n bits already set?
-// (Used to check if a memory range is already committed)
-static inline bool mi_bitmap_is_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
-  return mi_bitmap_is_xsetN(MI_BIT_SET, bitmap, idx, n);
-}
-
-// Is a sequence of n bits already clear?
-static inline bool mi_bitmap_is_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
-  return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, idx, n);
-}
-
-static inline bool mi_bitmap_is_set(mi_bitmap_t* bitmap, size_t idx) {
-  return mi_bitmap_is_setN(bitmap, idx, 1);
-}
-
-static inline bool mi_bitmap_is_clear(mi_bitmap_t* bitmap, size_t idx) {
-  return mi_bitmap_is_clearN(bitmap, idx, 1);
-}
-
-// Called once a bit is cleared to see if the memory slice can be claimed.
-typedef bool (mi_claim_fun_t)(size_t slice_index, mi_arena_t* arena, bool* keep_set);
-
-// Find a set bits in the bitmap, atomically clear it, and check if `claim` returns true.
-// If not claimed, continue on (potentially setting the bit again depending on `keep_set`).
-// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
-mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,
-                                                    mi_claim_fun_t* claim, mi_arena_t* arena );
-
-
-// Atomically clear a bit but only if it is set. Will block otherwise until the bit is set.
-// This is used to delay free-ing a page that it at the same time being considered to be
-// allocated from `mi_arena_try_abandoned` (and is in the `claim` function of `mi_bitmap_try_find_and_claim`).
-void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx);
-
-
-// If a bit is set in the bitmap, return `true` and set `idx` to the index of the highest bit.
-// Otherwise return `false` (and `*idx` is undefined).
-// Used for unloading arena's
-bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx);
-
-// Return count of all set bits in a bitmap.
-size_t mi_bitmap_popcount(mi_bitmap_t* bitmap);
-
-
-typedef bool (mi_forall_set_fun_t)(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg2);
-
-// Visit all set bits in a bitmap (`slice_count == 1`)
-bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
-
-// Visit all set bits in a bitmap with larger ranges if possible (`slice_count >= 1`)
-// Ranges will never cross chunk boundaries though (and `slice_count <= MI_BCHUNK_BITS`)
-bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
-
-// Visit all set bits in a bitmap with at least `rngslices` at a time (and aligned to `rngslices`). 
-// This is used by purging to not break up transparent huge pages for example.
-// Ranges will never cross chunk boundaries (and `slice_count <= MI_BCHUNK_BITS`).
-bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
-
-// Count all set bits in given range in the bitmap.
-size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n);
-
-/* ----------------------------------------------------------------------------
-  Binned concurrent bitmap
-  Assigns a size class to each chunk such that small blocks don't cause too
-  much fragmentation since we keep chunks for larger blocks separate.
----------------------------------------------------------------------------- */
-
-// mi_chunkbin_t is defined in mimalloc-stats.h
-
-static inline mi_chunkbin_t mi_chunkbin_inc(mi_chunkbin_t bbin) {
-  mi_assert_internal(bbin < MI_CBIN_COUNT);
-  return (mi_chunkbin_t)((int)bbin + 1);
-}
-
-static inline mi_chunkbin_t mi_chunkbin_dec(mi_chunkbin_t bbin) {
-  mi_assert_internal(bbin > MI_CBIN_NONE);
-  return (mi_chunkbin_t)((int)bbin - 1);
-}
-
-static inline mi_chunkbin_t mi_chunkbin_of(size_t slice_count) {
-  if (slice_count==1) return MI_CBIN_SMALL;
-  if (slice_count==8) return MI_CBIN_MEDIUM;
-  #if MI_ENABLE_LARGE_PAGES
-  if (slice_count==MI_BFIELD_BITS) return MI_CBIN_LARGE;
-  #endif
-  if (slice_count > MI_BCHUNK_BITS) return MI_CBIN_HUGE;
-  return MI_CBIN_OTHER;
-}
-
-// An atomic "binned" bitmap for the free slices where we keep chunks reserved for particular size classes
-typedef mi_decl_bchunk_align struct mi_bbitmap_s {
-  _Atomic(size_t)  chunk_count;         // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)
-  _Atomic(size_t)  chunk_max_accessed;  // max chunk index that was once cleared or set
-  #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 2
-  size_t           _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 2];    // suppress warning on msvc by aligning manually
-  #endif
-  mi_bchunkmap_t   chunkmap;
-  mi_bchunkmap_t   chunkmap_bins[MI_CBIN_COUNT - 1];             // chunkmaps with bit set if the chunk is in that size class (excluding MI_CBIN_NONE)
-  mi_bchunk_t      chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT];        // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT
-} mi_bbitmap_t;
-
-
-static inline size_t mi_bbitmap_chunk_count(const mi_bbitmap_t* bbitmap) {
-  return mi_atomic_load_relaxed(&((mi_bbitmap_t*)bbitmap)->chunk_count);
-}
-
-static inline size_t mi_bbitmap_max_bits(const mi_bbitmap_t* bbitmap) {
-  return (mi_bbitmap_chunk_count(bbitmap) * MI_BCHUNK_BITS);
-}
-
-mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunk_t* chunkmap_bins, size_t chunk_idx);
-
-size_t mi_bbitmap_size(size_t bit_count, size_t* chunk_count);
-
-// If a bit is clear in the bitmap, return `true` and set `idx` to the index of the highest bit that was clear.
-// Otherwise return `false` (and `*idx` is undefined).
-// Used for debug output.
-bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx);
-
-// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true
-// returns the size of the bitmap.
-size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero);
-
-// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).
-// Not atomic so only use if still local to a thread.
-void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
-
-
-// Set a sequence of `n` bits in the bbitmap; returns `true` if atomically transitioned from all 0's to 1's
-bool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
-
-
-// Is a sequence of n bits already all set/cleared?
-bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n);
-
-// Is a sequence of n bits already set?
-// (Used to check if a memory range is already committed)
-static inline bool mi_bbitmap_is_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
-  return mi_bbitmap_is_xsetN(MI_BIT_SET, bbitmap, idx, n);
-}
-
-// Is a sequence of n bits already clear?
-static inline bool mi_bbitmap_is_clearN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
-  return mi_bbitmap_is_xsetN(MI_BIT_CLEAR, bbitmap, idx, n);
-}
-
-
-// Try to atomically transition `n` bits from all set to all clear. Returns `true` on succes.
-// `n` cannot cross chunk boundaries, where `n <= MI_CHUNK_BITS`.
-bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
-
-
-// Specialized versions for common bit sequence sizes
-bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx);  // 1-bit
-bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 8-bits
-// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // MI_BFIELD_BITS
-bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // < MI_BFIELD_BITS
-bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS
-bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BCHUNK_BITS
-
-// Find a sequence of `n` bits in the bbitmap with all bits set, and try to atomically clear all.
-// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
-mi_decl_nodiscard static inline bool mi_bbitmap_try_find_and_clearN(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
-  if (n==1) return mi_bbitmap_try_find_and_clear(bbitmap, tseq, pidx);               // small pages
-  if (n==8) return mi_bbitmap_try_find_and_clear8(bbitmap, tseq, pidx);              // medium pages
-  // if (n==MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearX(bbitmap, tseq, pidx); // large pages
-  if (n==0) return false;
-  if (n<=MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearNX(bbitmap, tseq, n, pidx);
-  if (n<=MI_BCHUNK_BITS) return mi_bbitmap_try_find_and_clearNC(bbitmap, tseq, n, pidx);
-  return mi_bbitmap_try_find_and_clearN_(bbitmap, tseq, n, pidx);
-}
-
-
-#endif // MI_BITMAP_H
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/free.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/free.c
deleted file mode 100644
index 6f33340d..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/free.c
+++ /dev/null
@@ -1,658 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#if !defined(MI_IN_ALLOC_C)
-#error "this file should be included from 'alloc.c' (so aliases can work from alloc-override)"
-// add includes help an IDE
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"   // _mi_prim_thread_id()
-#endif
-
-// forward declarations
-static void   mi_check_padding(const mi_page_t* page, const mi_block_t* block);
-static bool   mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block);
-static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded);
-static void   mi_stat_free(const mi_page_t* page, const mi_block_t* block);
-
-
-// ------------------------------------------------------
-// Free
-// ------------------------------------------------------
-
-// regular free of a (thread local) block pointer
-// fast path written carefully to prevent spilling on the stack
-static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full)
-{
-  MI_UNUSED(was_guarded);
-  // checks  
-  if mi_unlikely(mi_check_is_double_free(page, block)) return;
-  if (!was_guarded) { mi_check_padding(page, block); }
-  if (track_stats) { mi_stat_free(page, block); }
-  #if (MI_DEBUG>0) && !MI_TRACK_ENABLED  && !MI_TSAN
-  memset(block, MI_DEBUG_FREED, mi_page_block_size(page));
-  #endif
-  if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned
-
-  // actual free: push on the local free list
-  mi_block_set_next(page, block, page->local_free);
-  page->local_free = block;
-  if mi_unlikely(--page->used == 0) {
-    if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page)
-      _mi_page_retire(page); 
-    }
-  }
-  else if mi_unlikely(check_full && mi_page_is_in_full(page)) {
-    _mi_page_unfull(page);
-  }
-}
-
-// Forward declaration for multi-threaded collect
-static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept;
-
-// Free a block multi-threaded
-static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded) mi_attr_noexcept
-{
-  MI_UNUSED(was_guarded);
-  // adjust stats (after padding check and potentially recursive `mi_free` above)
-  mi_stat_free(page, block);    // stat_free may access the padding
-  mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded));
-
-  // _mi_padding_shrink(page, block, sizeof(mi_block_t));
-#if (MI_DEBUG>0) && !MI_TRACK_ENABLED  && !MI_TSAN       // note: when tracking, cannot use mi_usable_size with multi-threading
-  if (!was_guarded) {
-    size_t dbgsize = mi_usable_size(block);
-    if (dbgsize > MI_MiB) { dbgsize = MI_MiB; }
-    _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);
-  }
-#endif
-
-  // push atomically on the page thread free list
-  mi_thread_free_t tf_new;
-  mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);
-  do {
-    mi_block_set_next(page, block, mi_tf_block(tf_old));
-    tf_new = mi_tf_create(block, true /* always use owned: try to claim it if the page is abandoned */);
-  } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough?
-
-  // and atomically try to collect the page if it was abandoned
-  const bool is_owned_now = !mi_tf_is_owned(tf_old);
-  if (is_owned_now) {
-    mi_assert_internal(mi_page_is_abandoned(page));
-    mi_free_try_collect_mt(page,block);
-  }
-}
-
-
-// Adjust a block that was allocated aligned, to the actual start of the block in the page.
-// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the
-// `page_start` and `block_size` fields; however these are constant and the page won't be
-// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently.
-mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
-  mi_assert_internal(page!=NULL && p!=NULL);
-
-  const size_t diff = (uint8_t*)p - mi_page_start(page);
-  const size_t block_size = mi_page_block_size(page);
-  const size_t adjust = (_mi_is_power_of_two(block_size) ? diff & (block_size - 1) : diff % block_size);
-  return (mi_block_t*)((uintptr_t)p - adjust);
-}
-
-// forward declaration for a MI_GUARDED build
-#if MI_GUARDED
-static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration
-static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
-  if (mi_block_ptr_is_guarded(block, p)) { 
-    mi_block_unguard(page, block, p); 
-    return true;
-  }
-  else {
-    return false;
-  }
-}
-#else
-static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
-  MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);
-  return false;
-}
-#endif
-
-static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, void* p ) {
-  mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer
-  #if MI_SECURE > 0
-  // in secure mode we always unalign to guard against free-ing interior pointers
-  return _mi_page_ptr_unalign(page,p);
-  #else
-  MI_UNUSED(page);
-  return (mi_block_t*)p;
-  #endif
-}
-
-
-// free a local pointer  (page parameter comes first for better codegen)
-static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept {
-  mi_assert_internal(p!=NULL && page != NULL);
-  mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
-  const bool was_guarded = mi_block_check_unguard(page, block, p);
-  mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */);
-}
-
-// free a pointer owned by another thread (page parameter comes first for better codegen)
-static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p) mi_attr_noexcept {
-  mi_assert_internal(p!=NULL && page != NULL);
-  mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
-  const bool was_guarded = mi_block_check_unguard(page, block, p);
-  mi_free_block_mt(page, block, was_guarded);
-}
-
-// generic free (for runtime integration)
-void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {
-  if (is_local) mi_free_generic_local(page,p);
-           else mi_free_generic_mt(page,p);
-}
-
-
-// Get the page belonging to a pointer
-// Does further checks in debug mode to see if this was a valid pointer.
-static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg)
-{
-  MI_UNUSED_RELEASE(msg);
-  #if MI_DEBUG
-  if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) {
-    _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p);
-    return NULL;
-  }
-  mi_page_t* page = _mi_safe_ptr_page(p);
-  if (p != NULL && page == NULL) {
-    _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p);
-  }
-  return page;
-  #else
-  return _mi_ptr_page(p);
-  #endif
-}
-
-// Free a block
-// Fast path written carefully to prevent register spilling on the stack
-static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* page)  
-{
-  if mi_unlikely(page==NULL) return;  // page will be NULL if p==NULL
-  mi_assert_internal(p!=NULL && page!=NULL);
-  if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
-
-  const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page));
-  if mi_likely(xtid == 0) {                        // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0`
-    // thread-local, aligned, and not a full page
-    mi_block_t* const block = mi_validate_block_from_ptr(page,p);
-    mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */);
-  }
-  else if (xtid <= MI_PAGE_FLAG_MASK) {            // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0`
-    // page is local, but is full or contains (inner) aligned blocks; use generic path
-    mi_free_generic_local(page, p);
-  }
-  // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap)
-  else if ((xtid & MI_PAGE_FLAG_MASK) == 0) {      // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0`
-    // blocks are aligned (and not a full page); push on the thread_free list
-    mi_block_t* const block = mi_validate_block_from_ptr(page,p);
-    mi_free_block_mt(page,block,false /* was_guarded */);
-  }
-  else {
-    // page is full or contains (inner) aligned blocks; use generic multi-thread path
-    mi_free_generic_mt(page, p);
-  }
-}
-
-void mi_free(void* p) mi_attr_noexcept {
-  mi_page_t* const page = mi_validate_ptr_page(p,"mi_free");  
-  mi_free_ex(p, NULL, page);
-}
-
-void mi_ufree(void* p, size_t* usable) mi_attr_noexcept {
-  mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree");  
-  mi_free_ex(p, usable, page);
-}
-
-void mi_free_small(void* p) mi_attr_noexcept {
-  // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`.
-  // If we keep page info in front of the page area for small objects, we can find the info
-  // just by aligning down the pointer instead of looking it up in the page map.
-  #if MI_PAGE_META_ALIGNED_FREE_SMALL 
-    #if MI_GUARDED 
-    #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as MI_GUARDED is defined"
-    mi_free(p);
-    #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE
-    #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size"
-    mi_free(p);
-    #else
-      mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE);
-      mi_assert(page == mi_validate_ptr_page(p,"mi_free_small"));
-      mi_assert((void*)page == _mi_align_down_ptr(page->page_start,MI_SMALL_PAGE_SIZE));
-      mi_assert(page->block_size <= MI_SMALL_SIZE_MAX);  // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small`
-      mi_free_ex(p, NULL, page);
-    #endif
-  #else
-  mi_free(p);
-  #endif  
-}
-
-
-// --------------------------------------------------------------------------------------------
-// `mi_free_try_collect_mt`: Potentially collect a page in a free in an abandoned page.
-// 1. if the page becomes empty, free it
-// 2. if it can be reclaimed, reclaim it in our theap
-// 3. if it went to < 7/8th used, re-abandon to be mapped (so it can be found by theaps looking for free pages)
-// --------------------------------------------------------------------------------------------
-
-// Helper for mi_free_try_collect_mt: free if the page has no more used blocks (this is updated by `_mi_page_free_collect(_partly)`)
-static bool mi_abandoned_page_try_free(mi_page_t* page)
-{
-  if (!mi_page_all_free(page)) return false;
-  // first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish)
-  _mi_arenas_page_unabandon(page,NULL);
-  _mi_arenas_page_free(page,NULL); // we can now free the page directly
-  return true;
-}
-
-// Helper for mi_free_try_collect_mt: try if we can reabandon a previously abandoned mostly full page to be mapped
-static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page)
-{
-  // if the page is unmapped, try to reabandon so it can possibly be mapped and found for allocations
-  // We only reabandon if a full page starts to have enough blocks available to prevent immediate re-abandon of a full page
-  if (mi_page_is_mostly_used(page)) return false;   // not too full
-  if (page->memid.memkind != MI_MEM_ARENA || mi_page_is_abandoned_mapped(page)) return false;  // and not already mapped (or unmappable)
-
-  mi_assert(!mi_page_is_full(page));
-  return _mi_arenas_page_try_reabandon_to_mapped(page);
-}
-
-// Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently
-// freed in the meantime. Returns `true` if the page was freed.
-// By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`.
-static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) {
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(!mi_page_all_free(page));
-  // try to cas atomically the original free list (`mt_free`) back with the ownership cleared.
-  mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true);
-  mi_thread_free_t tf_new    = mi_tf_create(expected_thread_free, false);
-  while mi_unlikely(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_expect, tf_new)) {
-    mi_assert_internal(mi_tf_is_owned(tf_expect));
-    // while the xthread_free list is not empty..
-    while (mi_tf_block(tf_expect) != NULL) {
-      // if there were concurrent updates to the thread-free list, we retry to free or reabandon to mapped (if it became !mosty_used).
-      _mi_page_free_collect(page,false);  // update used count
-      if (mi_abandoned_page_try_free(page)) return;
-      if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
-      // otherwise continue un-owning
-      tf_expect = mi_atomic_load_relaxed(&page->xthread_free);
-    }
-    // and try again to release ownership
-    mi_assert_internal(mi_tf_block(tf_expect)==NULL);
-    tf_new = mi_tf_create(NULL, false);
-  }
-}
-
-static inline bool mi_page_queue_len_is_atmost( mi_theap_t* theap, size_t block_size, long atmost) {
-  if (atmost < 0) return false;
-  mi_page_queue_t* const pq = mi_page_queue(theap,block_size);
-  mi_assert_internal(pq!=NULL);
-  return (pq->count <= (size_t)atmost);
-}
-
-// Helper for mi_free_try_collect_mt:  try to reclaim the page for ourselves
-static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long reclaim_on_free) mi_attr_noexcept
-{
-  // note: reclaiming can improve benchmarks like `larson` or `rbtree-ck` a lot even in the single-threaded case,
-  // since free-ing from an owned page avoids atomic operations. However, if we reclaim too eagerly in
-  // a multi-threaded scenario we may start to hold on to too much memory and reduce reuse among threads.
-  // If the current theap is where the page originally came from, we reclaim much more eagerly while
-  // 'cross-thread' reclaiming on free is by default off (and we only 'reclaim' these by finding the abandoned
-  // pages when we allocate a fresh page).
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(!mi_page_all_free(page));
-  mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE);
-  mi_assert_internal(reclaim_on_free >= 0);
-
-  // dont reclaim if we just have terminated this thread and we should
-  // not reinitialize the theap for this thread. (can happen due to thread-local destructors for example -- issue #944)
-  if (!_mi_thread_is_initialized()) return false;
-
-  // get our theap 
-  mi_theap_t* const theap = _mi_page_associated_theap_peek(page);
-  if (theap==NULL || !theap->allow_page_reclaim) return false;
-  
-  // todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance?
-  // set max_reclaim limit
-  long max_reclaim = 0;
-  if mi_likely(theap == page->theap) {  // did this page originate from the current theap? (and thus allocated from this thread)
-    // originating theap
-    max_reclaim = _mi_option_get_fast(theap->tld->is_in_threadpool ? mi_option_page_cross_thread_max_reclaim : mi_option_page_max_reclaim);
-  }
-  else if (reclaim_on_free == 1 &&               // if cross-thread is allowed
-            !theap->tld->is_in_threadpool &&      // and we are not part of a threadpool
-            !mi_page_is_mostly_used(page) &&     // and the page is not too full
-            _mi_arena_memid_is_suitable(page->memid, _mi_theap_heap(theap)->exclusive_arena)) {   // and it fits our memory
-    // across threads
-    max_reclaim = _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim);
-  }
-
-  // are we within the reclaim limit?
-  if (max_reclaim >= 0 && !mi_page_queue_len_is_atmost(theap, page->block_size, max_reclaim)) {
-    return false;
-  }
-
-  // reclaim the page into this theap
-  // first remove it from the abandoned pages in the arena -- this might wait for any readers to finish
-  _mi_arenas_page_unabandon(page, theap);
-  _mi_theap_page_reclaim(theap, page);
-  mi_theap_stat_counter_increase(theap, pages_reclaim_on_free, 1);
-  return true;
-}
-
-
-// We freed a block in an abandoned page (that was not owned). Try to collect
-static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept
-{
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(mi_page_is_abandoned(page));
-  mi_assert_internal(mt_free != NULL);
-  // we own the page now, and it is safe to collect the thread atomic free list
-  if (page->block_size <= MI_SMALL_SIZE_MAX) {
-    // use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list
-    // (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted).
-    //  however, if the page became completely free, the used count is guaranteed to be 0.)
-    mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used.
-    _mi_page_free_collect_partly(page, mt_free);    
-  }
-  else {
-    // for larger blocks we use the regular collect 
-    _mi_page_free_collect(page,false /* no force */);
-    mt_free = NULL; // expected page->xthread_free value after collection
-  }
-  const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free);
-  #if MI_DEBUG > 1
-  if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); }
-  if (mi_page_is_full(page))      { mi_assert(mi_page_is_mostly_used(page)); }
-  #endif
-
-  // try to: 1. free it, 2. reclaim it, or 3. reabandon it to be mapped
-  if (mi_abandoned_page_try_free(page)) return;
-  if (page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE && reclaim_on_free >= 0) {  // early test for better codegen
-    if (mi_abandoned_page_try_reclaim(page, reclaim_on_free)) return;
-  }
-  if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
-  
-  // otherwise unown the page again
-  mi_abandoned_page_unown_from_free(page, mt_free);
-}
-
-
-// ------------------------------------------------------
-// Usable size
-// ------------------------------------------------------
-
-// Bytes available in a block
-static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept {
-  const mi_block_t* block = _mi_page_ptr_unalign(page, p);
-  const bool is_guarded = mi_block_ptr_is_guarded(block,p);
-  const size_t size = mi_page_usable_size_of(page, block, is_guarded);
-  const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;
-  mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
-  const size_t aligned_size = (size - adjust);  
-  return aligned_size;
-}
-
-static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept {
-  if mi_unlikely(page==NULL) return 0;
-  if mi_likely(!mi_page_has_interior_pointers(page)) {
-    const mi_block_t* block = (const mi_block_t*)p;
-    return mi_page_usable_size_of(page, block, false /* is guarded */);
-  }
-  else {
-    // split out to separate routine for improved code generation
-    return mi_page_usable_aligned_size_of(page, p);
-  }
-}
-
-mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept {
-  const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size");
-  return _mi_usable_size(p,page);
-}
-
-
-// ------------------------------------------------------
-// Free variants
-// ------------------------------------------------------
-
-void mi_free_size(void* p, size_t size) mi_attr_noexcept {
-  MI_UNUSED_RELEASE(size);
-  #if MI_DEBUG
-  const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size");  
-  const size_t available = _mi_usable_size(p,page);
-  mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ );
-  #endif
-  mi_free(p);
-}
-
-void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {
-  MI_UNUSED_RELEASE(alignment);
-  mi_assert(((uintptr_t)p % alignment) == 0);
-  mi_free_size(p,size);
-}
-
-void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
-  MI_UNUSED_RELEASE(alignment);
-  mi_assert(((uintptr_t)p % alignment) == 0);
-  mi_free(p);
-}
-
-
-// ------------------------------------------------------
-// Check for double free in secure and debug mode
-// This is somewhat expensive so only enabled for secure mode 4
-// ------------------------------------------------------
-
-#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))
-// linear check if the free list contains a specific element
-static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) {
-  while (list != NULL) {
-    if (elem==list) return true;
-    list = mi_block_next(page, list);
-  }
-  return false;
-}
-
-static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) {
-  // The decoded value is in the same page (or NULL).
-  // Walk the free lists to verify positively if it is already freed
-  if (mi_list_contains(page, page->free, block) ||
-      mi_list_contains(page, page->local_free, block) ||
-      mi_list_contains(page, mi_page_thread_free(page), block))
-  {
-    _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page));
-    return true;
-  }
-  return false;
-}
-
-#define mi_track_page(page,access)  { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); }
-
-static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
-  bool is_double_free = false;
-  mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field
-  if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 &&  // quick check: aligned pointer?
-      (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL?
-  {
-    // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free?
-    // (continue in separate function to improve code generation)
-    is_double_free = mi_check_is_double_freex(page, block);
-  }
-  return is_double_free;
-}
-#else
-static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
-  MI_UNUSED(page);
-  MI_UNUSED(block);
-  return false;
-}
-#endif
-
-
-// ---------------------------------------------------------------------------
-// Check for theap block overflow by setting up padding at the end of the block
-// ---------------------------------------------------------------------------
-
-#if MI_PADDING // && !MI_TRACK_ENABLED
-static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) {
-  *bsize = mi_page_usable_block_size(page);
-  const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);
-  mi_track_mem_defined(padding,sizeof(mi_padding_t));
-  *delta = padding->delta;
-  uint32_t canary = padding->canary;
-  uintptr_t keys[2];
-  keys[0] = page->keys[0];
-  keys[1] = page->keys[1];
-  bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize);
-  mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
-  return ok;
-}
-
-// Return the exact usable size of a block.
-static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
-  if (is_guarded) {
-    const size_t bsize = mi_page_block_size(page);
-    return (bsize - _mi_os_page_size());
-  }
-  else {
-    size_t bsize;
-    size_t delta;
-    bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
-    mi_assert_internal(ok); mi_assert_internal(delta <= bsize);
-    return (ok ? bsize - delta : 0);
-  }
-}
-
-// When a non-thread-local block is freed, it becomes part of the thread delayed free
-// list that is freed later by the owning theap. If the exact usable size is too small to
-// contain the pointer for the delayed list, then shrink the padding (by decreasing delta)
-// so it will later not trigger an overflow error in `mi_free_block`.
-void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
-  size_t bsize;
-  size_t delta;
-  bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
-  mi_assert_internal(ok);
-  if (!ok || (bsize - delta) >= min_size) return;  // usually already enough space
-  mi_assert_internal(bsize >= min_size);
-  if (bsize < min_size) return;  // should never happen
-  size_t new_delta = (bsize - min_size);
-  mi_assert_internal(new_delta < bsize);
-  mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize);
-  mi_track_mem_defined(padding,sizeof(mi_padding_t));
-  padding->delta = (uint32_t)new_delta;
-  mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
-}
-#else
-static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
-  MI_UNUSED(is_guarded); MI_UNUSED(block);
-  return mi_page_usable_block_size(page);
-}
-
-void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
-  MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(min_size);
-}
-#endif
-
-#if MI_PADDING && MI_PADDING_CHECK
-
-static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) {
-  size_t bsize;
-  size_t delta;
-  bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
-  *size = *wrong = bsize;
-  if (!ok) return false;
-  mi_assert_internal(bsize >= delta);
-  *size = bsize - delta;
-  if (!mi_page_is_huge(page)) {
-    uint8_t* fill = (uint8_t*)block + bsize - delta;
-    const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes
-    mi_track_mem_defined(fill, maxpad);
-    for (size_t i = 0; i < maxpad; i++) {
-      if (fill[i] != MI_DEBUG_PADDING) {
-        *wrong = bsize - delta + i;
-        ok = false;
-        break;
-      }
-    }
-    mi_track_mem_noaccess(fill, maxpad);
-  }
-  return ok;
-}
-
-static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
-  size_t size;
-  size_t wrong;
-  if (!mi_verify_padding(page,block,&size,&wrong)) {
-    _mi_error_message(EFAULT, "buffer overflow in theap block %p of size %zu: write after %zu bytes\n", block, size, wrong );
-  }
-}
-
-#else
-
-static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
-  MI_UNUSED(page);
-  MI_UNUSED(block);
-}
-
-#endif
-
-// only maintain stats for smaller objects if requested
-#if (MI_STAT>0)
-static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
-  MI_UNUSED(block);
-  mi_theap_t* const theap = _mi_theap_default();
-  if (!mi_theap_is_initialized(theap)) return; // (for now) skip statistics if free'd after thread_done was called (usually a thread cleanup call by the OS)
-
-  const size_t bsize = mi_page_usable_block_size(page);
-  // #if (MI_STAT>1)
-  // const size_t usize = mi_page_usable_size_of(page, block);
-  // mi_theap_stat_decrease(theap, malloc_requested, usize);
-  // #endif
-  if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
-    mi_theap_stat_decrease(theap, malloc_normal, bsize);
-    #if (MI_STAT > 1)
-    mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1);
-    #endif
-  }
-  else {
-    const size_t bpsize = mi_page_block_size(page);  // match stat in page.c:mi_huge_page_alloc
-    mi_theap_stat_decrease(theap, malloc_huge, bpsize);
-  }
-}
-#else
-void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
-  MI_UNUSED(page); MI_UNUSED(block);
-}
-#endif
-
-
-// Remove guard page when building with MI_GUARDED
-#if MI_GUARDED
-static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {
-  MI_UNUSED(p);
-  mi_assert_internal(mi_block_ptr_is_guarded(block, p));
-  mi_assert_internal(mi_page_has_interior_pointers(page));
-  mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));
-  mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);
-
-  const size_t bsize = mi_page_block_size(page);
-  const size_t psize = _mi_os_page_size();
-  mi_assert_internal(bsize > psize);
-  mi_assert_internal(!page->memid.is_pinned);
-  void* gpage = (uint8_t*)block + bsize - psize;
-  mi_assert_internal(_mi_is_aligned(gpage, psize));
-  _mi_os_unprotect(gpage, psize);
-}
-#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/heap.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/heap.c
deleted file mode 100644
index f0a01652..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/heap.c
+++ /dev/null
@@ -1,273 +0,0 @@
-/*----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"  // _mi_theap_default
-
-
-/* -----------------------------------------------------------
-  Heap's
------------------------------------------------------------ */
-
-mi_theap_t* mi_heap_theap(mi_heap_t* heap) {
-  return _mi_heap_theap(heap);
-}
-
-void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node) {
-  if (heap==NULL) { heap = mi_heap_main(); }
-  heap->numa_node = (numa_node < 0 ? -1 : numa_node % _mi_os_numa_node_count());
-}
-
-void mi_heap_stats_merge_to_subproc(mi_heap_t* heap) {
-  if (heap==NULL) { heap = mi_heap_main(); }
-  _mi_stats_merge_into(&heap->subproc->stats, &heap->stats);
-}
-
-void mi_heap_stats_merge_to_main(mi_heap_t* heap) {
-  if (heap==NULL) return;
-  _mi_stats_merge_into(&mi_heap_main()->stats, &heap->stats);
-}
-
-static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_heap)
-{
-  mi_heap_t* heap = (mi_heap_t*)const_heap;
-  mi_assert_internal(heap!=NULL);
-
-  if (_mi_is_heap_main(heap)) {
-    // this can be called if the (main) thread is not yet initialized (as no allocation happened)
-    // but `theap_main_init_get()` will call `mi_thread_init()`
-    mi_theap_t* const theap = _mi_theap_main_safe();
-    mi_assert_internal(theap!=NULL && _mi_is_heap_main(_mi_theap_heap(theap)));
-    return theap;
-  }
-
-  // otherwise initialize the theap for this heap
-  // get the thread local
-  mi_assert_internal(heap->theap != 0);
-  if (heap->theap==0) {  // paranoia
-    _mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap);
-    return NULL;
-  }
-  mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);
-
-  // create a fresh theap?
-  if (theap==NULL) {
-    // set first an invalid value to ensure the thread local storage is allocated
-    if (!_mi_thread_local_set(heap->theap, (mi_theap_t*)1)) {
-      _mi_error_message(EFAULT, "unable to allocate memory for thread local storage\n");
-      return NULL;
-    }    
-    // then allocate the theap
-    theap = _mi_theap_create(heap, _mi_theap_default_safe()->tld);
-    _mi_thread_local_set(heap->theap, theap);  // Cannot fail now as it was set before. Always set so the local is valid or NULL (and not 1)
-    if (theap==NULL) {
-      _mi_error_message(EFAULT, "unable to allocate memory for a thread local heap\n");
-      return NULL;
-    }    
-  }
-  return theap;
-}
-
-
-// get the theap for a heap without initializing (and return NULL in that case)
-mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) {
-  if (heap==NULL || _mi_is_heap_main(heap)) {
-    return _mi_theap_main_safe(); 
-  }
-  else {
-    return (mi_theap_t*)_mi_thread_local_get(heap->theap);
-  }
-}
-
-// get (and possibly create) the theap belonging to a heap
-mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap)
-{
-  mi_theap_t* theap = _mi_heap_theap_peek(heap);
-  if mi_unlikely(theap==NULL) {
-    theap = mi_heap_init_theap(heap);
-    if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; }  // this will return NULL from page.c:_mi_malloc_generic
-  }
-  _mi_theap_cached_set(theap);
-  return theap;
-}
-
-
-mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {
-  // always allocate heap data in the (subprocess) main heap
-  mi_heap_t* const heap_main = mi_heap_main();
-  // todo: allocate heap data in the exclusive arena ?
-  mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) );
-  if (heap==NULL) return NULL;
-
-  // reserve a thread local slot for this heap (see also issue #1230)
-  const mi_thread_local_t theap_slot = _mi_thread_local_create();
-  if (theap_slot == 0) {
-    _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n");
-    mi_free(heap);
-    return NULL;
-  }
-
-  // init fields
-  heap->theap = theap_slot;
-  heap->subproc = heap_main->subproc;
-  heap->heap_seq = mi_atomic_increment_relaxed(&heap_main->subproc->heap_total_count);
-  heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id);
-  heap->numa_node = -1; // no initial affinity
-  mi_stats_header_init(&heap->stats);
-  mi_lock_init(&heap->theaps_lock);
-  mi_lock_init(&heap->os_abandoned_pages_lock);
-  mi_lock_init(&heap->arena_pages_lock);
-
-  // push onto the subproc heaps
-  mi_lock(&heap->subproc->heaps_lock) {
-    mi_heap_t* head = heap->subproc->heaps;
-    heap->prev = NULL;
-    heap->next = head;
-    if (head!=NULL) { head->prev = heap;  }
-    heap->subproc->heaps = heap;
-  }
-  mi_atomic_increment_relaxed(&heap_main->subproc->heap_count);
-  mi_subproc_stat_increase(heap_main->subproc, heaps, 1);
-  return heap;
-}
-
-mi_heap_t* mi_heap_new(void) {
-  return mi_heap_new_in_arena(0);
-}
-
-// free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency)
-static void mi_heap_free_theaps(mi_heap_t* heap) {
-  // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`), 
-  // and we need to ensure we free theaps atomically.
-  // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock 
-  // potential concurrent thread termination which tries to remove the theap from our theaps list.
-  bool all_freed;
-  do {
-    all_freed = true;
-    mi_theap_t* theap = NULL;
-    mi_lock(&heap->theaps_lock) { 
-      theap = heap->theaps; 
-      while(theap != NULL) {
-        mi_theap_t* next = theap->hnext;
-        if (!_mi_theap_free(theap, false /* dont re-acquire the heap->theaps_lock */, true /* acquire the tld->theaps_lock though */ )) {
-          all_freed = false;
-        }
-        theap = next;
-      }      
-    }
-    if (!all_freed) { 
-      mi_heap_stat_counter_increase(heap,heaps_delete_wait,1); 
-      _mi_prim_thread_yield();
-    }
-    else { 
-      mi_assert_internal(heap->theaps==NULL); 
-    }               
-  }
-  while(!all_freed);
-}
-
-// free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed)
-static void mi_heap_free(mi_heap_t* heap) {
-  mi_assert_internal(heap!=NULL && !_mi_is_heap_main(heap));
-
-  // free all arena pages infos
-  mi_lock(&heap->arena_pages_lock) {
-    for (size_t i = 0; i < MI_MAX_ARENAS; i++) {
-      mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
-      if (arena_pages!=NULL) {
-        mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);
-        mi_free(arena_pages);
-      }
-    }
-  }
-
-  // remove the heap from the subproc
-  mi_heap_stats_merge_to_main(heap);
-  mi_atomic_decrement_relaxed(&heap->subproc->heap_count);
-  mi_subproc_stat_decrease(heap->subproc, heaps, 1);
-  mi_lock(&heap->subproc->heaps_lock) {
-    if (heap->next!=NULL) { heap->next->prev = heap->prev; }
-    if (heap->prev!=NULL) { heap->prev->next = heap->next; }
-                     else { heap->subproc->heaps = heap->next; }
-  }
-
-  _mi_thread_local_free(heap->theap);
-  mi_lock_done(&heap->theaps_lock);
-  mi_lock_done(&heap->os_abandoned_pages_lock);
-  mi_lock_done(&heap->arena_pages_lock);
-  mi_free(heap);
-}
-
-void mi_heap_delete(mi_heap_t* heap) {
-  if (heap==NULL) return;
-  if (_mi_is_heap_main(heap)) {
-    _mi_warning_message("cannot delete the main heap\n");
-    return;
-  }
-  mi_heap_free_theaps(heap);
-  _mi_heap_move_pages(heap, mi_heap_main());
-  mi_heap_free(heap);
-}
-
-void _mi_heap_force_destroy(mi_heap_t* heap) {
-  if (heap==NULL) return;
-  mi_heap_free_theaps(heap);
-  _mi_heap_destroy_pages(heap);
-  if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); }  // todo: release locks of the main heap?
-}
-
-void mi_heap_destroy(mi_heap_t* heap) {
-  if (heap==NULL) return;
-  if (_mi_is_heap_main(heap)) {
-    _mi_warning_message("cannot destroy the main heap\n");
-    return;
-  }
-  _mi_heap_force_destroy(heap);
-}
-
-mi_heap_t* mi_heap_of(const void* p) {
-  mi_page_t* page = _mi_safe_ptr_page(p);
-  if (page==NULL) return NULL;
-  return mi_page_heap(page);
-}
-
-bool mi_any_heap_contains(const void* p) {
-  return (mi_heap_of(p)!=NULL);
-}
-
-bool mi_heap_contains(const mi_heap_t* heap, const void* p) {
-  if (heap==NULL) { heap = mi_heap_main(); }
-  return (heap==mi_heap_of(p));
-}
-
-// deprecated
-bool mi_check_owned(const void* p) {
-  return mi_any_heap_contains(p);
-}
-
-// unsafe heap utilization function for DragonFly (see issue #1258)
-// If the page of pointer `p` belongs to `heap` (or `heap==NULL`) and has less than `perc_threshold` used blocks in its used area return `true`.
-// This function is unsafe in general as it assumes we are the only thread accessing the page of `p`.
-bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept {
-  if (p==NULL) return false;
-  const mi_page_t* const page = _mi_safe_ptr_page(p);   // Get the page containing this pointer
-  if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false;
-  // If the page is the head of the queue, it is currently being used for 
-  // allocations; we skip it to avoid immediate thrashing.
-  if (page->prev == NULL)  return false;
-
-  // match heap?
-  const mi_heap_t* const page_heap = mi_page_heap(page);
-  if (page_heap==NULL) return false;
-  if (heap!=NULL && page_heap!=heap) return false;
-    
-  // check utilization
-  if (page->capacity==0)   return false;
-  if (perc_threshold>=100) return true;
-  return (perc_threshold >= ((100UL*page->used) / page->capacity));
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/init.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/init.c
deleted file mode 100644
index 1fabd891..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/init.c
+++ /dev/null
@@ -1,1199 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"
-
-#include <string.h>  // memcpy, memset
-#include <stdlib.h>  // atexit
-
-#define MI_MEMID_INIT(kind)   {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ }
-#define MI_MEMID_STATIC       MI_MEMID_INIT(MI_MEM_STATIC)
-
-// Empty page used to initialize the small free pages array
-const mi_page_t _mi_page_empty = {
-  MI_ATOMIC_VAR_INIT(0),  // xthread_id
-  NULL,                   // free
-  0,                      // used
-  0,                      // capacity
-  0,                      // reserved capacity
-  0,                      // retire_expire
-  false,                  // is_zero
-  NULL,                   // local_free
-  MI_ATOMIC_VAR_INIT(0),  // xthread_free
-  0,                      // block_size
-  NULL,                   // page_start
-  #if (MI_PADDING || MI_ENCODE_FREELIST)
-  { 0, 0 },               // keys
-  #endif
-  NULL,                   // theap
-  NULL,                   // heap
-  NULL, NULL,             // next, prev
-  MI_ARENA_SLICE_SIZE,    // page_committed
-  MI_MEMID_STATIC         // memid
-};
-
-#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)
-
-#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8)
-#define MI_SMALL_PAGES_EMPTY  { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
-#elif (MI_PADDING>0)
-#define MI_SMALL_PAGES_EMPTY  { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
-#else
-#define MI_SMALL_PAGES_EMPTY  { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }
-#endif
-
-
-// Empty page queues for every bin
-#define QNULL(sz)  { NULL, NULL, 0, (sz)*sizeof(uintptr_t) }
-#define MI_PAGE_QUEUES_EMPTY \
-  { QNULL(1), \
-    QNULL(     1), QNULL(     2), QNULL(     3), QNULL(     4), QNULL(     5), QNULL(     6), QNULL(     7), QNULL(     8), /* 8 */ \
-    QNULL(    10), QNULL(    12), QNULL(    14), QNULL(    16), QNULL(    20), QNULL(    24), QNULL(    28), QNULL(    32), /* 16 */ \
-    QNULL(    40), QNULL(    48), QNULL(    56), QNULL(    64), QNULL(    80), QNULL(    96), QNULL(   112), QNULL(   128), /* 24 */ \
-    QNULL(   160), QNULL(   192), QNULL(   224), QNULL(   256), QNULL(   320), QNULL(   384), QNULL(   448), QNULL(   512), /* 32 */ \
-    QNULL(   640), QNULL(   768), QNULL(   896), QNULL(  1024), QNULL(  1280), QNULL(  1536), QNULL(  1792), QNULL(  2048), /* 40 */ \
-    QNULL(  2560), QNULL(  3072), QNULL(  3584), QNULL(  4096), QNULL(  5120), QNULL(  6144), QNULL(  7168), QNULL(  8192), /* 48 */ \
-    QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \
-    QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \
-    QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \
-    QNULL(MI_LARGE_MAX_OBJ_WSIZE + 1  /* 655360, Huge queue */), \
-    QNULL(MI_LARGE_MAX_OBJ_WSIZE + 2) /* Full queue */ }
-
-#define MI_STAT_COUNT_NULL()  {0,0,0}
-
-// Empty statistics
-#define MI_STAT_COUNT(stat)     {0,0,0},
-#define MI_STAT_COUNTER(stat)   {0},
-
-#define MI_STATS_NULL  \
-  MI_STAT_FIELDS() \
-  \
-  { MI_INIT4(MI_STAT_COUNT_NULL) }, \
-  { { 0 }, { 0 }, { 0 }, { 0 } }, \
-  \
-  { MI_INIT74(MI_STAT_COUNT_NULL) }, \
-  { MI_INIT74(MI_STAT_COUNT_NULL) }, \
-  { MI_INIT5(MI_STAT_COUNT_NULL) }
-
-// --------------------------------------------------------
-// Statically allocate an empty theap as the initial
-// thread local value for the default theap,
-// and statically allocate the backing theap for the main
-// thread so it can function without doing any allocation
-// itself (as accessing a thread local for the first time
-// may lead to allocation itself on some platforms)
-// --------------------------------------------------------
-
-static mi_decl_cache_align mi_subproc_t subproc_main
-#if __cplusplus
-  = { };     // empty initializer to prevent running the constructor (with msvc)
-#else
-  = { 0 };   // C zero initialize
-#endif
-
-static mi_subproc_t* subprocs = &subproc_main;
-static mi_lock_t     subprocs_lock;
-
-static mi_decl_cache_align mi_tld_t tld_empty = {
-  0,                      // thread_id
-  0,                      // thread_seq
-  0,                      // default numa node
-  &subproc_main,          // subproc
-  NULL,                   // theaps list
-  MI_LOCK_INITIALIZER,    // theaps lock
-  false,                  // recurse
-  false,                  // is_in_threadpool
-  MI_MEMID_STATIC         // memid
-};
-
-mi_decl_cache_align const mi_theap_t _mi_theap_empty = {
-  &tld_empty,             // tld
-  MI_ATOMIC_VAR_INIT(NULL), // heap
-  MI_ATOMIC_VAR_INIT(1),  // refcount
-  0,                      // heartbeat
-  0,                      // cookie
-  { {0}, {0}, 0, true },  // random
-  0,                      // page count
-  MI_BIN_FULL, 0,         // page retired min/max
-  0,                      // pages_full_size
-  0, 0,                   // generic count
-  NULL, NULL,             // tnext, tprev
-  NULL, NULL,             // hnext, hprev
-  0,                      // full page retain
-  false,                  // allow reclaim
-  true,                   // allow abandon
-  #if MI_GUARDED
-  0, 0, 0, 1,             // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)
-  #endif
-  MI_SMALL_PAGES_EMPTY,
-  MI_PAGE_QUEUES_EMPTY,
-  MI_MEMID_STATIC,
-  { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL },      // stats
-};
-
-mi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = {
-  &tld_empty,             // tld
-  MI_ATOMIC_VAR_INIT(NULL), // heap
-  MI_ATOMIC_VAR_INIT(1),  // refcount
-  0,                      // heartbeat
-  0,                      // cookie
-  { {0}, {0}, 0, true },  // random
-  0,                      // page count
-  MI_BIN_FULL, 0,         // page retired min/max
-  0,                      // pages_full_size
-  0, 0,                   // generic count
-  NULL, NULL,             // tnext, tprev
-  NULL, NULL,             // hnext, hprev
-  0,                      // full page retain
-  false,                  // allow reclaim
-  true,                   // allow abandon
-  #if MI_GUARDED
-  0, 0, 0, 1,             // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)
-  #endif
-  MI_SMALL_PAGES_EMPTY,
-  MI_PAGE_QUEUES_EMPTY,
-  MI_MEMID_STATIC,
-  { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL },      // stats
-};
-
-// Heap for the main thread
-
-extern mi_decl_hidden mi_decl_cache_align mi_theap_t theap_main;
-extern mi_decl_hidden mi_decl_cache_align mi_heap_t  heap_main;
-
-static mi_decl_cache_align mi_tld_t tld_main = {
-  0,                      // thread_id
-  0,                      // thread_seq
-  0,                      // numa node
-  &subproc_main,          // subproc
-  &theap_main,            // theaps list
-  MI_LOCK_INITIALIZER,    // theaps lock
-  false,                  // recurse
-  false,                  // is_in_threadpool
-  MI_MEMID_STATIC         // memid
-};
-
-mi_decl_cache_align mi_theap_t theap_main = {
-  &tld_main,              // thread local data
-  MI_ATOMIC_VAR_INIT(&heap_main), // main heap
-  MI_ATOMIC_VAR_INIT(1),  // refcount
-  0,                      // heartbeat
-  0,                      // initial cookie
-  { {0x846ca68b}, {0}, 0, true },  // random
-  0,                      // page count
-  MI_BIN_FULL, 0,         // page retired min/max
-  0,                      // pages_full_size
-  0, 0,                   // generic count
-  NULL, NULL,             // tnext, tprev
-  NULL, NULL,             // hnext, hprev
-  2,                      // full page retain
-  true,                   // allow page reclaim
-  true,                   // allow page abandon
-  #if MI_GUARDED
-  0, 0, 0, 0,
-  #endif
-  MI_SMALL_PAGES_EMPTY,
-  MI_PAGE_QUEUES_EMPTY,
-  MI_MEMID_STATIC,
-  { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL },      // stats
-};
-
-mi_decl_cache_align mi_heap_t heap_main
-#if __cplusplus
-  = { };     // empty initializer to prevent running the constructor (with msvc)
-#else
-  = { 0 };   // C zero initialize
-#endif
-
-// the theap belonging to the main heap
-mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main = NULL;
-
-mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
-  mi_threadid_t tid = _mi_prim_thread_id();
-  mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits
-  return tid;
-}
-
-#if MI_TLS_MODEL_THREAD_LOCAL
-// the thread-local main theap for allocation
-mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty;
-// the last used non-main theap
-mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty;
-#endif
-
-bool _mi_process_is_initialized = false;  // set to `true` in `mi_process_init`.
-
-mi_stats_t _mi_stats_main = { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL };
-
-#undef MI_STAT_COUNT
-#undef MI_STAT_COUNTER
-
-
-#if MI_GUARDED
-mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {
-  theap->guarded_sample_rate  = sample_rate;
-  theap->guarded_sample_count = sample_rate;  // count down samples
-  if (theap->guarded_sample_rate > 1) {
-    if (seed == 0) {
-      seed = _mi_theap_random_next(theap);
-    }
-    theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1;  // start at random count between 1 and `sample_rate`
-  }
-}
-
-mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {
-  theap->guarded_size_min = min;
-  theap->guarded_size_max = (min > max ? min : max);
-}
-
-void _mi_theap_guarded_init(mi_theap_t* theap) {
-  mi_theap_guarded_set_sample_rate(theap,
-    (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX),
-    (size_t)mi_option_get(mi_option_guarded_sample_seed));
-  mi_theap_guarded_set_size_bound(theap,
-    (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX),
-    (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) );
-}
-#else
-mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {
-  MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed);
-}
-
-mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {
-  MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max);
-}
-void _mi_theap_guarded_init(mi_theap_t* theap) {
-  MI_UNUSED(theap);
-}
-#endif
-
-/* -----------------------------------------------------------
-  Initialization
-  Note: on some platforms lock_init or just a thread local access
-  can cause allocation and induce recursion during initialization.
------------------------------------------------------------ */
-
-
-// Initialize main subproc
-static void mi_subproc_main_init(void) {
-  if (subproc_main.memid.memkind != MI_MEM_STATIC) {
-    subproc_main.memid = _mi_memid_create(MI_MEM_STATIC);
-    subproc_main.heaps = &heap_main;
-    subproc_main.heap_total_count = 1;
-    subproc_main.heap_count = 1;
-    mi_atomic_store_ptr_release(mi_heap_t, &subproc_main.heap_main, &heap_main);
-    __mi_stat_increase_mt(&subproc_main.stats.heaps, 1);
-    mi_stats_header_init(&subproc_main.stats);
-    mi_lock_init(&subproc_main.arena_reserve_lock);
-    mi_lock_init(&subproc_main.heaps_lock);
-    mi_lock_init(&subprocs_lock);
-    mi_lock_init(&tld_empty.theaps_lock); 
-  }
-}
-
-// Initialize main tld
-static void mi_tld_main_init(void) {
-  if (tld_main.thread_id == 0) {
-    tld_main.thread_id = _mi_prim_thread_id();
-    mi_lock_init(&tld_main.theaps_lock);
-  }
-}
-
-void _mi_theap_options_init(mi_theap_t* theap) {
-  theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0);
-  theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0);
-  theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);
-}
-
-// Initialization of the (statically allocated) main theap, and the main tld and subproc.
-static void mi_theap_main_init(void) {
-  if mi_unlikely(theap_main.memid.memkind != MI_MEM_STATIC) {
-    // theap
-    theap_main.memid = _mi_memid_create(MI_MEM_STATIC);
-    #if defined(__APPLE__) || defined(_WIN32) && !defined(MI_SHARED_LIB)
-      _mi_random_init_weak(&theap_main.random);    // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)
-    #else
-      _mi_random_init(&theap_main.random);
-    #endif
-    theap_main.cookie  = _mi_theap_random_next(&theap_main);
-    _mi_theap_options_init(&theap_main);
-    _mi_theap_guarded_init(&theap_main);
-  }
-}
-
-// Initialize main heap
-static void mi_heap_main_init(void) {
-  if mi_unlikely(heap_main.subproc == NULL) {
-    heap_main.subproc = &subproc_main;
-    heap_main.theaps = &theap_main;
-
-    mi_theap_main_init();
-    mi_subproc_main_init();
-    mi_tld_main_init();
-
-    mi_lock_init(&heap_main.theaps_lock);
-    mi_lock_init(&heap_main.os_abandoned_pages_lock);
-    mi_lock_init(&heap_main.arena_pages_lock);
-  }
-}
-
-
-/* -----------------------------------------------------------
-  Thread local data
------------------------------------------------------------ */
-
-// Allocate fresh tld
-static mi_tld_t* mi_tld_alloc(void) {
-  if (_mi_is_main_thread()) {
-    mi_atomic_increment_relaxed(&tld_main.subproc->thread_count);
-    return &tld_main;
-  }
-  else {
-    // allocate tld meta-data
-    // note: we need to be careful to not access the tld from `_mi_meta_zalloc`
-    // (and in turn from `_mi_arena_alloc_aligned` and `_mi_os_alloc_aligned`).
-    mi_memid_t memid;
-    mi_tld_t* tld = (mi_tld_t*)_mi_meta_zalloc(sizeof(mi_tld_t), &memid);
-    if (tld==NULL) {
-      _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n");
-      return NULL;
-    }
-    tld->memid = memid;
-    tld->theaps = NULL;
-    mi_lock_init(&tld->theaps_lock);
-    tld->subproc = &subproc_main;
-    tld->numa_node = _mi_os_numa_node();
-    tld->thread_id = _mi_prim_thread_id();
-    tld->thread_seq = mi_atomic_increment_relaxed(&tld->subproc->thread_total_count);
-    tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool();
-    mi_atomic_increment_relaxed(&tld->subproc->thread_count);
-    return tld;
-  }
-}
-
-#define MI_TLD_INVALID  ((mi_tld_t*)1)
-
-mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) {
-  mi_lock_done(&tld->theaps_lock);
-  if (tld != NULL && tld != MI_TLD_INVALID) {
-    mi_atomic_decrement_relaxed(&tld->subproc->thread_count);
-    _mi_meta_free(tld, sizeof(mi_tld_t), tld->memid);
-  }
-  #if 0
-  // do not read/write to `thread_tld` on older macOS <= 14 as that will re-initialize the thread local storage
-  // (since we are calling this during pthread shutdown)
-  // (and this could happen on other systems as well, so let's never do it)
-  thread_tld = MI_TLD_INVALID;
-  #endif
-}
-
-// return the thread local heap ensuring it is initialized (and not `NULL` or `&_mi_theap_empty`);
-mi_theap_t* _mi_theap_default_safe(void) {
-  mi_theap_t* theap = _mi_theap_default();
-  if mi_likely(mi_theap_is_initialized(theap)) return theap;
-  mi_thread_init();
-  mi_assert_internal(mi_theap_is_initialized(_mi_theap_default()));
-  return _mi_theap_default();
-}
-
-// return the main theap ensuring it is initialized. 
-mi_theap_t* _mi_theap_main_safe(void) {
-  mi_theap_t* theap = __mi_theap_main;
-  if mi_unlikely(theap==NULL) {  // if thread_init or default_set was never called
-    mi_thread_init();            // sets the default slot to the main theap
-    theap = _mi_theap_default();
-    mi_assert_internal(theap!=NULL);
-    mi_assert_internal(_mi_is_theap_main(theap));
-    if (_mi_is_theap_main(theap)) {
-      __mi_theap_main = theap;
-    }
-  }
-  mi_assert_internal(theap!=NULL && _mi_is_theap_main(theap));    
-  return theap;
-}
-
-
-mi_subproc_t* _mi_subproc_main(void) {
-  return &subproc_main;
-}
-
-mi_subproc_t* _mi_subproc(void) {
-  // should work without doing initialization (as it may be called from `_mi_tld -> mi_tld_alloc ... -> os_alloc -> _mi_subproc()`
-  // todo: this will still fail on OS systems where the first access to a thread-local causes allocation.
-  //       on such systems we can check for this with the _mi_prim_get_default_theap as those are protected (by being
-  //       stored in a TLS slot for example)
-  mi_theap_t* theap = _mi_theap_default();
-  if (theap == NULL) {
-    return _mi_subproc_main();
-  }
-  else {
-    return theap->tld->subproc;  // avoid using thread local storage (`thread_tld`)
-  }
-}
-
-mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) {
-  mi_heap_t* heap = mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);
-  if mi_likely(heap!=NULL) {
-    return heap;
-  }
-  else {
-    mi_heap_main_init();
-    mi_assert_internal(mi_atomic_load_relaxed(&subproc->heap_main) != NULL);
-    return mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);
-  }
-}
-
-mi_heap_t* mi_heap_main(void) {
-  return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init
-}
-
-bool _mi_is_heap_main(const mi_heap_t* heap) {
-  mi_assert_internal(heap!=NULL);
-  return (_mi_subproc_heap_main(heap->subproc) == heap);
-}
-
-bool _mi_is_theap_main(const mi_theap_t* theap) {
-  return (mi_theap_is_initialized(theap) && _mi_is_heap_main(_mi_theap_heap(theap)));
-}
-
-/* -----------------------------------------------------------
-  Sub process
------------------------------------------------------------ */
-
-
-mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {
-  return (mi_subproc_t*)(subproc_id._mi_subproc_id);
-}
-
-mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) {
-  mi_subproc_id_t id = { subproc };
-  return id;
-}
-
-mi_subproc_id_t mi_subproc_main(void) {
-  return _mi_subproc_to_id(_mi_subproc_main());
-}
-
-mi_subproc_id_t mi_subproc_current(void) {
-  return _mi_subproc_to_id(_mi_subproc());
-}
-
-mi_subproc_id_t mi_subproc_new(void) {
-  static _Atomic(size_t) subproc_total_count;
-  mi_memid_t memid;
-  mi_subproc_t* subproc = (mi_subproc_t*)_mi_meta_zalloc(sizeof(mi_subproc_t),&memid);
-  if (subproc == NULL) return _mi_subproc_to_id(NULL);
-  subproc->memid = memid;
-  subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count) + 1;
-  mi_stats_header_init(&subproc->stats);
-  mi_lock_init(&subproc->arena_reserve_lock);
-  mi_lock_init(&subproc->heaps_lock);
-  mi_lock(&subprocs_lock) {
-    // push on subproc list
-    subproc->next = subprocs;
-    if (subprocs!=NULL) { subprocs->prev = subproc; }
-    subprocs = subproc;
-  }
-  return _mi_subproc_to_id(subproc);
-}
-
-// destroy all subproc resources including arena's, heap's etc.
-static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock)
-{
-  if (subproc==NULL) return;
-
-  // remove from the subproc list
-  mi_lock_maybe(&subprocs_lock, acquire_subprocs_lock) {
-    if (subproc->next!=NULL) { subproc->next->prev = subproc->prev;  }
-    if (subproc->prev!=NULL) { subproc->prev->next = subproc->next;  }
-                        else { mi_assert_internal(subprocs==subproc);  subprocs = subproc->next; }
-  }
-
-  // destroy all subproc heaps
-  mi_lock(&subproc->heaps_lock) {
-    mi_heap_t* heap = subproc->heaps;
-    while (heap != NULL) {
-      mi_heap_t* next = heap->next;
-      if (heap!=subproc->heap_main) { mi_heap_destroy(heap); }
-      heap = next;
-    }
-    mi_assert_internal(subproc->heaps == subproc->heap_main);
-    _mi_heap_force_destroy(subproc->heap_main);  // no warning if destroying the main heap
-  }
-
-  // remove associated arenas
-  _mi_arenas_unsafe_destroy_all(subproc);
-
-  // merge stats back into the main subproc?
-  if (subproc!=&subproc_main) {
-    _mi_stats_merge_into(&subproc_main.stats, &subproc->stats);
-  }
-
-  // safe to release
-  // todo: should we refcount subprocesses?
-  mi_lock_done(&subproc->arena_reserve_lock);
-  mi_lock_done(&subproc->heaps_lock);
-  if (subproc!=&subproc_main) {
-    _mi_meta_free(subproc, sizeof(mi_subproc_t), subproc->memid);
-  }
-  else {
-    // for the main subproc, also release the global page map
-    _mi_page_map_unsafe_destroy(&subproc_main);
-  }
-}
-
-void mi_subproc_destroy(mi_subproc_id_t subproc_id) {
-  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
-  if (subproc==NULL || subproc==&subproc_main) return;
-  mi_subproc_unsafe_destroy(subproc, true /* take lock */);
-}
-
-static void mi_subprocs_unsafe_destroy_all(void) {
-  mi_lock(&subprocs_lock) {
-    mi_subproc_t* subproc = subprocs;
-    while (subproc!=NULL) {
-      mi_subproc_t* next = subproc->next;
-      if (subproc!=&subproc_main) {
-        mi_subproc_unsafe_destroy(subproc, false /* take subprocs lock */);
-      }
-      subproc = next;
-    }
-  }  
-  mi_subproc_unsafe_destroy(&subproc_main, true /* take subprocs lock */);
-}
-
-
-void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {
-  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
-  mi_tld_t* const tld = _mi_theap_default_safe()->tld;
-  mi_assert(tld->subproc== &subproc_main);
-  if (tld->subproc != &subproc_main) {
-    _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (id: %p)\n", subproc);
-    return;
-  }
-  tld->subproc = subproc;
-  tld->thread_seq = mi_atomic_increment_relaxed(&subproc->thread_total_count);
-  mi_atomic_decrement_relaxed(&subproc_main.thread_count);
-  mi_atomic_increment_relaxed(&subproc->thread_count);
-}
-
-
-bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) {
-  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
-  if (subproc==NULL) return false;
-  bool ok = true;
-  mi_lock(&subproc->heaps_lock) {
-    for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) {
-      ok = (*visitor)(heap, arg);
-    }
-  }
-  return ok;
-}
-
-
-/* -----------------------------------------------------------
-  Allocate theap data
------------------------------------------------------------ */
-
-// Initialize the thread local default theap, called from `mi_thread_init`
-static mi_theap_t* _mi_thread_init_theap_default(void) {
-  mi_theap_t* theap = _mi_theap_default();
-  if (mi_theap_is_initialized(theap)) return theap;
-  if (_mi_is_main_thread()) {
-    mi_heap_main_init();
-    theap = &theap_main;
-  }
-  else {
-    // allocates tld data
-    // note: we cannot access thread-locals yet as that can cause (recursive) allocation
-    // (on macOS <= 14 for example where the loader allocates thread-local data on demand).
-    mi_tld_t* tld = mi_tld_alloc();
-    if (tld==NULL) return NULL;  // things are very wrong if this fails (out of memory)
-    // allocate and initialize the theap for the main heap
-    theap = _mi_theap_create(mi_heap_main(), tld);
-  }
-  // associate the theap with this thread
-  // (this is safe, on macOS for example, the theap is set in a dedicated TLS slot and thus does not cause recursive allocation)
-  _mi_theap_default_set(theap);
-  return theap;
-}
-
-
-// Free the thread local theaps
-static void mi_thread_theaps_done(mi_tld_t* tld)
-{
-  // reset the thread local theaps
-  _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty);
-  _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty);
-  __mi_theap_main = NULL;
-  
-  // abandon the pages of all theaps in this thread
-  mi_lock(&tld->theaps_lock) {
-    mi_theap_t* theap = tld->theaps;
-    while (theap != NULL) {
-      mi_theap_t* next = theap->tnext; 
-      // never destroy theaps; if a dll is linked statically with mimalloc,
-      // there may still be delete/free calls after the mi_fls_done is called. Issue #207
-      _mi_theap_collect_abandon(theap);
-      mi_assert_internal(theap->page_count==0);
-      theap = next;
-    }
-  }
-
-  // free the theaps of this thread.
-  // This can run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically.
-  // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock 
-  // potential concurrent `mi_heap_free_theaps` which tries to remove the theap from our theaps list.
-  bool all_freed;
-  do {
-    all_freed = true;
-    mi_lock(&tld->theaps_lock) {
-      mi_theap_t* theap = tld->theaps;
-      while (theap != NULL) {
-        mi_theap_t* next = theap->tnext;
-        mi_assert_internal(theap->page_count==0);
-        if (!_mi_theap_free(theap, true /* acquire heap->theaps_lock */, false /* dont re-acquire the tld->theaps_lock*/ )) {
-          all_freed = false;
-        }
-        theap = next;
-      }
-    }
-    if (!all_freed) { 
-      mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1); 
-      _mi_prim_thread_yield(); 
-    }
-    else { 
-      mi_assert_internal(tld->theaps==NULL); 
-    }
-  } while (!all_freed);
-
-  mi_assert(_mi_theap_default()==(mi_theap_t*)&_mi_theap_empty); // careful to not re-initialize the default theap during theap_delete
-  mi_assert(!mi_theap_is_initialized(_mi_theap_default()));
-}
-
-
-
-// --------------------------------------------------------
-// Try to run `mi_thread_done()` automatically so any memory
-// owned by the thread but not yet released can be abandoned
-// and re-owned by another thread.
-//
-// 1. windows dynamic library:
-//     call from DllMain on DLL_THREAD_DETACH
-// 2. windows static library:
-//     use special linker section to call a destructor when the thread is done
-// 3. unix, pthreads:
-//     use a pthread key to call a destructor when a pthread is done
-//
-// In the last two cases we also need to call `mi_process_init`
-// to set up the thread local keys.
-// --------------------------------------------------------
-
-// Set up handlers so `mi_thread_done` is called automatically
-static void mi_process_setup_auto_thread_done(void) {
-  mi_atomic_do_once {
-    _mi_prim_thread_init_auto_done();
-    _mi_theap_default_set(&theap_main);
-  }
-}
-
-
-bool _mi_is_main_thread(void) {
-  return (tld_main.thread_id==0 || tld_main.thread_id == _mi_thread_id());
-}
-
-
-// Initialize thread
-void mi_thread_init(void) mi_attr_noexcept
-{
-  // ensure our process has started already
-  mi_process_init();
-  // if the theap_default is already set we have already initialized
-  if (_mi_thread_is_initialized()) return;
-
-  // initialize the default theap
-  if (_mi_thread_init_theap_default() == NULL) return; // out-of-memory on tld/theap allocation
-
-  mi_heap_stat_increase(mi_heap_main(), threads, 1);
-  // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
-}
-
-void mi_thread_done(void) mi_attr_noexcept {
-  _mi_thread_done(NULL);
-}
-
-void _mi_thread_done(mi_theap_t* _theap_main)
-{
-  // NULL can be passed on some platforms
-  if (_theap_main==NULL) {
-    _theap_main = __mi_theap_main;  // don't call `mi_theap_main_safe` as that re-initializes the thread
-    if (_theap_main==NULL) {        // can happen if `mi_theap_main_safe` is never called; but then the default is main
-      _theap_main = _mi_theap_default();
-      mi_assert_internal(_theap_main==NULL || _mi_is_theap_main(_theap_main));
-    }
-  }
-
-  // prevent re-entrancy through theap_done/theap_set_default_direct (issue #699)
-  if (!mi_theap_is_initialized(_theap_main)) {
-    return;
-  }
-
-  // release dynamic thread_local's
-  _mi_thread_locals_thread_done();
-
-  // note: we store the tld as we should avoid reading `thread_tld` at this point (to avoid reinitializing the thread local storage)
-  mi_tld_t* const tld = _theap_main->tld;
-
-  // adjust stats
-  mi_heap_stat_decrease(_mi_subproc_heap_main(tld->subproc), threads, 1);  // todo: or `_theap_main->heap`?
-
-  // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local theaps...
-  if (tld->thread_id != _mi_prim_thread_id()) return;
-
-  // delete the thread local theaps
-  mi_thread_theaps_done(tld);
-
-  // free thread local data
-  mi_tld_free(tld);
-}
-
-
-mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) {
-  return (mi_theap_t*)&_mi_theap_empty;
-}
-
-#if MI_TLS_MODEL_DYNAMIC_WIN32
-
-// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed)
-// See <https://en.wikipedia.org/wiki/Win32_Thread_Information_Block> for the offsets.
-#if MI_SIZE_SIZE==4
-#define MI_TLS_DIRECT_FIRST             (0x0E10 / MI_SIZE_SIZE)
-#else
-#define MI_TLS_DIRECT_FIRST             (0x1480 / MI_SIZE_SIZE)
-#endif
-#define MI_TLS_DIRECT_SLOTS             (64)
-#define MI_TLS_EXPANSION_SLOTS          (1024)
-
-#if !MI_WIN_DIRECT_TLS
-#define MI_TLS_INITIAL_SLOT             MI_TLS_EXPANSION_SLOT
-#define MI_TLS_INITIAL_EXPANSION_SLOT   (MI_TLS_EXPANSION_SLOTS-1)
-#else
-// with only direct entries, use the "arbitrary user data" field 
-// and assume it is NULL (see also <http://www.nynaeve.net/?p=98>)
-#define MI_TLS_INITIAL_SLOT             (5)
-#define MI_TLS_INITIAL_EXPANSION_SLOT   (0)
-#endif
-
-// we initially use the last of the expansion slots as the default NULL.
-// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc (which is quite unlikely)
-mi_decl_hidden mi_decl_cache_align size_t _mi_theap_default_slot = MI_TLS_INITIAL_SLOT;
-mi_decl_hidden size_t _mi_theap_default_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT;
-mi_decl_hidden size_t _mi_theap_cached_slot            = MI_TLS_INITIAL_SLOT;
-mi_decl_hidden size_t _mi_theap_cached_expansion_slot  = MI_TLS_INITIAL_EXPANSION_SLOT;
-
-static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES;
-static DWORD mi_tls_raw_index_cached  = TLS_OUT_OF_INDEXES;
-
-static bool mi_win_tls_slot_alloc(size_t* slot, size_t* extended, DWORD* raw_index) {
-  const DWORD index = TlsAlloc();
-  *raw_index = index;
-  if (index==TLS_OUT_OF_INDEXES) {
-    *extended = 0;
-    *slot = 0;
-    return false;
-  }
-  else if (index<MI_TLS_DIRECT_SLOTS) {
-    *extended = 0;
-    *slot = index + MI_TLS_DIRECT_FIRST;
-    return true;
-  }
-  #if !MI_WIN_DIRECT_TLS
-  else if (index < MI_TLS_DIRECT_SLOTS + MI_TLS_EXPANSION_SLOTS - 1) { // check maximum number of expansion slots - 1 (as we use the last one as the default)    
-    *extended = index - MI_TLS_DIRECT_SLOTS;
-    *slot = MI_TLS_EXPANSION_SLOT;
-    return true;
-  }
-  #endif
-  else {
-    // to high an index for us
-    _mi_error_message(EFAULT, "returned tls index was too high (%u)\n", index);
-    TlsFree(index);
-    *raw_index = TLS_OUT_OF_INDEXES; 
-    *extended = 0;
-    *slot = 0;
-    return false;
-  }
-}
-
-static void mi_win_tls_slot_free(DWORD* raw_index) {
-  if (*raw_index != TLS_OUT_OF_INDEXES) {
-    TlsFree(*raw_index);
-    *raw_index = TLS_OUT_OF_INDEXES;
-  }
-}
-
-static void mi_tls_slots_init(void) {
-  mi_atomic_do_once {
-    bool ok = mi_win_tls_slot_alloc(&_mi_theap_default_slot, &_mi_theap_default_expansion_slot, &mi_tls_raw_index_default);
-    if (ok) {
-      ok = mi_win_tls_slot_alloc(&_mi_theap_cached_slot, &_mi_theap_cached_expansion_slot, &mi_tls_raw_index_cached);
-    }
-    if (!ok) {
-      _mi_error_message(EFAULT, "unable to allocate fast TLS user slot (0x%zx)\n", _mi_theap_cached_slot);
-    }
-  }
-}
-
-static void mi_tls_slots_done(void) {
-  mi_win_tls_slot_free(&mi_tls_raw_index_default);
-  mi_win_tls_slot_free(&mi_tls_raw_index_cached );
-}
-
-static void mi_win_tls_slot_set(size_t slot, size_t extended_slot, void* value) {
-  mi_assert_internal((slot >= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT);
-  if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) {
-    mi_prim_tls_slot_set(slot, value);
-  }
-  else {
-    mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS);
-    TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value);  // use TlsSetValue to initialize the TlsExpansion array if needed
-  }
-}
-
-#elif MI_TLS_MODEL_DYNAMIC_PTHREADS
-
-// only for pthreads for now
-mi_decl_hidden pthread_key_t _mi_theap_default_key = 0;
-mi_decl_hidden pthread_key_t _mi_theap_cached_key = 0;
-
-// create a non-zero pthread key
-static int mi_pthread_key_create( pthread_key_t* pkey ) {
-  pthread_key_t key;
-  int err = pthread_key_create(&key, NULL);
-  if (err!=0) return err;
-  if (key==0) {
-    // if we get a zero key, create another one as we use 0 for an invalid key
-    pthread_key_t key2;
-    err = pthread_key_create(&key2, NULL);
-    pthread_key_delete(key);  // delete the old key
-    if (err!=0) return err;
-    key = key2;
-  }
-  mi_assert_internal(key!=0);    
-  *pkey = key;
-  return 0;
-}
-
-static void mi_tls_slots_init(void) {
-  mi_atomic_do_once {
-    int err = mi_pthread_key_create(&_mi_theap_default_key);
-    if (err==0) {
-      err = mi_pthread_key_create(&_mi_theap_cached_key);
-    }
-    if (err!=0) {
-      _mi_error_message(EFAULT, "unable to allocate pthread keys (error %d)\n", err);
-    }
-  }
-}
-
-static void mi_tls_slots_done(void) {
-  if (_mi_theap_default_key != 0) {
-    pthread_key_delete(_mi_theap_default_key);
-    _mi_theap_default_key = 0;
-  }
-  if (_mi_theap_cached_key != 0) {
-    pthread_key_delete(_mi_theap_cached_key);
-    _mi_theap_cached_key = 0;
-  }
-}
-
-#else
-
-static void mi_tls_slots_init(void) {
-  // nothing
-}
-
-static void mi_tls_slots_done(void) {
-  // nothing
-}
-
-#endif
-
-void _mi_theap_cached_set(mi_theap_t* theap) {
-  mi_theap_t* prev = _mi_theap_cached();
-  if (prev==theap) return;
-  // set
-  mi_tls_slots_init();
-  #if MI_TLS_MODEL_THREAD_LOCAL
-    __mi_theap_cached = theap;
-  #elif MI_TLS_MODEL_FIXED_SLOT
-    mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_CACHED, theap);
-  #elif MI_TLS_MODEL_DYNAMIC_WIN32
-    mi_win_tls_slot_set(_mi_theap_cached_slot, _mi_theap_cached_expansion_slot, theap);
-  #elif MI_TLS_MODEL_DYNAMIC_PTHREADS
-    if (_mi_theap_cached_key!=0) pthread_setspecific(_mi_theap_cached_key, theap);
-  #endif
-  // update refcounts (so cached theap memory keeps available until no longer cached)
-  _mi_theap_incref(theap);
-  _mi_theap_decref(prev);    
-}
-
-void _mi_theap_default_set(mi_theap_t* theap)  {
-  mi_theap_t* const theap_old = _mi_theap_default();
-  mi_assert_internal(theap != NULL);
-  mi_assert_internal(theap->tld->thread_id==0 || theap->tld->thread_id==_mi_thread_id());
-  mi_tls_slots_init();
-  #if MI_TLS_MODEL_THREAD_LOCAL
-    __mi_theap_default = theap;
-  #elif MI_TLS_MODEL_FIXED_SLOT
-    mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_DEFAULT, theap);
-  #elif MI_TLS_MODEL_DYNAMIC_WIN32
-    mi_win_tls_slot_set(_mi_theap_default_slot, _mi_theap_default_expansion_slot, theap);
-  #elif MI_TLS_MODEL_DYNAMIC_PTHREADS
-    if (_mi_theap_default_key!=0) pthread_setspecific(_mi_theap_default_key, theap);
-  #endif
-
-  // set theap main if needed
-  if (mi_theap_is_initialized(theap)) {
-    // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again)
-    _mi_prim_thread_associate_default_theap(theap);
-    if (_mi_is_heap_main(_mi_theap_heap(theap))) {
-      __mi_theap_main = theap;
-    }
-  }
-
-  // ensure either the default slot contains the main theap, or __mi_theap_main is initialized 
-  if (mi_theap_is_initialized(theap_old) && _mi_is_heap_main(_mi_theap_heap(theap_old))) {
-    __mi_theap_main = theap_old;
-  }
-}
-
-void mi_thread_set_in_threadpool(void) mi_attr_noexcept {
-  mi_theap_t* theap = _mi_theap_default_safe();
-  theap->tld->is_in_threadpool = true;
-}
-
-// --------------------------------------------------------
-// Run functions on process init/done, and thread init/done
-// --------------------------------------------------------
-static bool os_preloading = true;    // true until this module is initialized
-
-// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.
-bool mi_decl_noinline _mi_preloading(void) {
-  return os_preloading;
-}
-
-// Returns true if mimalloc was redirected
-mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept {
-  return _mi_is_redirected();
-}
-
-// Called once by the process loader from `src/prim/prim.c`
-void _mi_auto_process_init(void) {
-  // mi_heap_main_init();
-  // #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)
-  // volatile mi_theap_t* dummy = __mi_theap_default; // access TLS to allocate it before setting tls_initialized to true;
-  // if (dummy == NULL) return;                       // use dummy or otherwise the access may get optimized away (issue #697)
-  // #endif
-
-  os_preloading = false;
-  mi_assert_internal(_mi_is_main_thread());
-
-  mi_process_init();
-  mi_process_setup_auto_thread_done();
-  _mi_thread_locals_init();
-  _mi_options_post_init();  // now we can print to stderr
-  if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n");
-
-  // show message from the redirector (if present)
-  const char* msg = NULL;
-  _mi_allocator_init(&msg);
-  if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {
-    _mi_fputs(NULL,NULL,NULL,msg);
-  }
-
-  // reseed random
-  _mi_random_reinit_if_weak(&theap_main.random);
-}
-
-// CPU features
-mi_decl_cache_align size_t _mi_cpu_movsb_max = 0;  // for size <= max, rep movsb is fast
-mi_decl_cache_align size_t _mi_cpu_stosb_max = 0;  // for size <= max, rep stosb is fast
-mi_decl_cache_align bool _mi_cpu_has_popcnt = false;
-
-#if (MI_ARCH_X64 || MI_ARCH_X86)
-#if defined(__GNUC__)
-// #include <cpuid.h>
-static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
-  // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx)
-  // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect).
-  uint32_t eax, ebx, ecx, edx;
-  __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : );
-  regs4[0] = eax;
-  regs4[1] = ebx;
-  regs4[2] = ecx;
-  regs4[3] = edx;
-  return true;
-}
-
-#elif defined(_MSC_VER)
-static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
-  __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel);
-  return true;
-}
-#else
-static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
-  MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel);
-  return false;
-}
-#endif
-
-static void mi_detect_cpu_features(void) {
-  // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))
-  // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?)
-  // FSRS for fast short rep stosb
-  bool amd = false;
-  bool fsrm = false;
-  // bool erms = false;
-  bool fsrs = false;
-  uint32_t cpu_info[4];
-  if (mi_cpuid(cpu_info, 0, 0)) {
-    amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD)
-  }
-  if (mi_cpuid(cpu_info, 7, 0)) {
-    fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
-    // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
-  }
-  if (mi_cpuid(cpu_info, 7, 1)) {
-    fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=1:_Extended_Features>
-  }
-  if (mi_cpuid(cpu_info, 1, 0)) {
-    _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see <https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits>
-  }
-
-  if (fsrm) {
-    _mi_cpu_movsb_max = 127;
-  }
-  if (fsrs || (amd && fsrm)) {  // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803
-    _mi_cpu_stosb_max = 127;
-  }
-}
-
-#else
-static void mi_detect_cpu_features(void) {
-  #if MI_ARCH_ARM64
-  _mi_cpu_has_popcnt = true;
-  #endif
-}
-#endif
-
-
-// Initialize the process; called by thread_init or the process loader
-static void mi_process_init_once(void) mi_attr_noexcept {
-  _mi_process_is_initialized = true;  
-  _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id());
-
-  mi_detect_cpu_features();
-  _mi_options_init();
-  _mi_stats_init();
-  _mi_os_init();
-  // the following can potentially allocate (on freeBSD for pthread keys)
-  // todo: do 2-phase so we can use stats at first, then later init the keys?
-  mi_heap_main_init(); // before page_map_init so stats are working
-  _mi_page_map_init(); // todo: this could fail.. should we abort in that case?
-  mi_thread_init();
-  _mi_process_is_initialized = true;
-
-  #if defined(_WIN32) && defined(MI_WIN_USE_FLS)
-  // On windows, when building as a static lib the FLS cleanup happens to early for the main thread.
-  // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup
-  // will not call _mi_thread_done on the (still executing) main thread. See issue #508.
-  _mi_prim_thread_associate_default_theap(NULL);
-  #endif
-
-  // mi_stats_reset();  // only call stat reset *after* thread init (or the theap tld == NULL)
-  mi_track_init();
-  if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
-    size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024);
-    int reserve_at  = (int)mi_option_get_clamp(mi_option_reserve_huge_os_pages_at, -1, INT_MAX);
-    if (reserve_at != -1) {
-      mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500);
-    } else {
-      mi_reserve_huge_os_pages_interleave(pages, 0, pages*500);
-    }
-  }
-  if (mi_option_is_enabled(mi_option_reserve_os_memory)) {
-    long ksize = mi_option_get(mi_option_reserve_os_memory);
-    if (ksize > 0) {
-      mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true);
-    }
-  }
-}
-
-// Initialize the process; called by thread_init or the process loader
-void mi_process_init(void) mi_attr_noexcept {
-  // #if _MSC_VER < 1920
-	// mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main
-	// #endif
-  mi_atomic_do_once {
-    mi_process_init_once();
-  }
-}
-
-// Called when the process is done (cdecl as it is used with `at_exit` on some platforms)
-void mi_cdecl mi_process_done(void) mi_attr_noexcept {
-  // only shutdown if we were initialized
-  if (!_mi_process_is_initialized) return;
-  // ensure we are called once
-  static bool process_done = false;
-  if (process_done) return;
-  process_done = true;
-
-  // free dynamic thread locals (if used at all)
-  _mi_thread_locals_done();
-
-  // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread
-  _mi_prim_thread_done_auto_done();
-
-  #ifndef MI_SKIP_COLLECT_ON_EXIT
-    #if (MI_DEBUG || !defined(MI_SHARED_LIB))
-    // free all memory if possible on process exit. This is not needed for a stand-alone process
-    // but should be done if mimalloc is statically linked into another shared library which
-    // is repeatedly loaded/unloaded, see issue #281.
-    mi_theap_collect(_mi_theap_default(), true /* force */);
-    #endif
-  #endif
-
-  // done with tracking tools
-  mi_track_done()
-
-  // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free
-  // since after process_done there might still be other code running that calls `free` (like at_exit routines,
-  // or C-runtime termination code.
-  if (mi_option_is_enabled(mi_option_destroy_on_exit)) {
-    mi_subprocs_unsafe_destroy_all(); // destroys all subprocs, arenas, and the page_map!    
-  }
-  else {
-    mi_heap_stats_merge_to_subproc(mi_heap_main());
-  }
-  
-  // careful now to no longer access any allocator functionality 
-  if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {
-    mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL);
-  }
-  mi_lock_done(&subprocs_lock);
-  mi_tls_slots_done();
-  _mi_allocator_done();
-  _mi_verbose_message("process done: 0x%zx\n", tld_main.thread_id);
-  os_preloading = true; // don't call the C runtime anymore
-}
-
-void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept {
-  if (_mi_option_get_fast(mi_option_destroy_on_exit)>1) return;
-  mi_process_done();
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/libc.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/libc.c
deleted file mode 100644
index c74003a1..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/libc.c
+++ /dev/null
@@ -1,477 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-// --------------------------------------------------------
-// This module defines various std libc functions to reduce
-// the dependency on libc, and also prevent errors caused
-// by some libc implementations when called before `main`
-// executes (due to malloc redirection)
-// --------------------------------------------------------
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"      // mi_prim_getenv
-
-char _mi_toupper(char c) {
-  if (c >= 'a' && c <= 'z') return (c - 'a' + 'A');
-                       else return c;
-}
-
-int _mi_strnicmp(const char* s, const char* t, size_t n) {
-  mi_assert_internal(s!=NULL && t!=NULL);
-  if (n == 0) return 0;
-  for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {
-    if (_mi_toupper(*s) != _mi_toupper(*t)) break;
-  }
-  return (n == 0 ? 0 : *s - *t);
-}
-
-bool _mi_streq(const char* s, const char* t) {
-  if (s==NULL && t==NULL) return true;
-  if (s==NULL || t==NULL) return false;
-  for (; *s != 0 && *t != 0; s++, t++) {
-    if (*s != *t) break;
-  }
-  return (*s == *t);
-}
-
-void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
-  if (dest==NULL || src==NULL || dest_size == 0) return;
-  // copy until end of src, or when dest is (almost) full
-  while (*src != 0 && dest_size > 1) {
-    *dest++ = *src++;
-    dest_size--;
-  }
-  // always zero terminate
-  *dest = 0;
-}
-
-void _mi_strlcat(char* dest, const char* src, size_t dest_size) {
-  if (dest==NULL || src==NULL || dest_size == 0) return;
-  // find end of string in the dest buffer
-  while (*dest != 0 && dest_size > 1) {
-    dest++;
-    dest_size--;
-  }
-  // and catenate
-  _mi_strlcpy(dest, src, dest_size);
-}
-
-size_t _mi_strnlen(const char* s, size_t max_len) {
-  if (s==NULL) return 0;
-  size_t len = 0;
-  while(s[len] != 0 && len < max_len) { len++; }
-  return len;
-}
-
-size_t _mi_strlen(const char* s) {
-  return _mi_strnlen(s,PTRDIFF_MAX);
-}
-
-char* _mi_strnstr(char* s, size_t max_len, const char* pat) {
-  if (s==NULL) return NULL;
-  if (pat==NULL) return s;
-  const size_t m = _mi_strnlen(s, max_len);
-  const size_t n = _mi_strlen(pat);  
-  for (size_t start = 0; start + n <= m; start++) {
-    size_t i = 0;
-    while (i<n && pat[i]==s[start+i]) {
-      i++;
-    }
-    if (i==n) return &s[start];
-  }
-  return NULL;
-}
-
-#ifdef MI_NO_GETENV
-bool _mi_getenv(const char* name, char* result, size_t result_size) {
-  MI_UNUSED(name);
-  MI_UNUSED(result);
-  MI_UNUSED(result_size);
-  return false;
-}
-#else
-bool _mi_getenv(const char* name, char* result, size_t result_size) {
-  if (name==NULL || result == NULL || result_size < 64) return false;
-  return _mi_prim_getenv(name,result,result_size);
-}
-#endif
-
-
-// --------------------------------------------------------
-// Define our own primitives for doing an action once
-// --------------------------------------------------------
-
-// Returns `true` only on the first invocation, signifying we can execute an action once.
-// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
-// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
-bool _mi_atomic_once_enter(mi_atomic_once_t* once) {
-  const uintptr_t once_tid = mi_atomic_load_acquire(&once->tid);
-  if mi_likely(once_tid == 1) {
-    return false; // already executed
-  }
-  const mi_threadid_t current_tid = _mi_thread_id();
-  if (once_tid == current_tid) {
-    return false; // recursive invocation; we need this for process_init for example
-  }  
-
-  mi_lock_acquire(&once->lock);
-  uintptr_t expected = 0;
-  if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) {  // could use atomic_load/store as well
-    return true;  // should execute and release
-  } 
-  else {
-    mi_lock_release(&once->lock);
-    return false; // already another thread entered and released
-  }
-}
-
-void _mi_atomic_once_release(mi_atomic_once_t* once) {
-  if (mi_atomic_load_acquire(&once->tid)>1) {  // paranoia
-    mi_atomic_store_release(&once->tid,1);     // done executing
-    mi_lock_release(&once->lock);
-  }  
-}
-
-
-// --------------------------------------------------------
-// Define our own limited `_mi_vsnprintf` and `_mi_snprintf`
-// This is mostly to avoid calling these when libc is not yet
-// initialized (and to reduce dependencies)
-//
-// format:      d i, p x u, s
-// prec:        z l ll L
-// width:       10
-// align-left:  -
-// fill:        0
-// plus:        +
-// --------------------------------------------------------
-
-static void mi_outc(char c, char** out, char* end) {
-  char* p = *out;
-  if (p >= end) return;
-  *p = c;
-  *out = p + 1;
-}
-
-static void mi_outs(const char* s, char** out, char* end) {
-  if (s == NULL) return;
-  char* p = *out;
-  while (*s != 0 && p < end) {
-    *p++ = *s++;
-  }
-  *out = p;
-}
-
-static void mi_out_fill(char fill, size_t len, char** out, char* end) {
-  char* p = *out;
-  for (size_t i = 0; i < len && p < end; i++) {
-    *p++ = fill;
-  }
-  *out = p;
-}
-
-static void mi_out_alignright(char fill, char* start, size_t len, size_t extra, char* end) {
-  if (len == 0 || extra == 0) return;
-  if (start + len + extra >= end) return;
-  // move `len` characters to the right (in reverse since it can overlap)
-  for (size_t i = 1; i <= len; i++) {
-    start[len + extra - i] = start[len - i];
-  }
-  // and fill the start
-  for (size_t i = 0; i < extra; i++) {
-    start[i] = fill;
-  }
-}
-
-
-static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* end)
-{
-  if (x == 0 || base == 0 || base > 16) {
-    if (prefix != 0) { mi_outc(prefix, out, end); }
-    mi_outc('0',out,end);
-  }
-  else {
-    // output digits in reverse
-    char* start = *out;
-    while (x > 0) {
-      char digit = (char)(x % base);
-      mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end);
-      x = x / base;
-    }
-    if (prefix != 0) {
-      mi_outc(prefix, out, end);
-    }
-    size_t len = *out - start;
-    // and reverse in-place
-    for (size_t i = 0; i < (len / 2); i++) {
-      char c = start[len - i - 1];
-      start[len - i - 1] = start[i];
-      start[i] = c;
-    }
-  }
-}
-
-
-#define MI_NEXTC()  c = *in; if (c==0) break; in++;
-
-int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
-  if (buf == NULL || bufsize == 0 || fmt == NULL) return 0;
-  buf[bufsize - 1] = 0;
-  char* const end = buf + (bufsize - 1);
-  const char* in = fmt;
-  char* out = buf;
-  while (true) {
-    if (out >= end) break;
-    char c;
-    MI_NEXTC();
-    if (c != '%') {
-      if (c == '\\') {
-        MI_NEXTC();
-        switch (c) {
-        case 'e': mi_outc('\x1B', &out, end); break;
-        case 't': mi_outc('\t', &out, end); break;
-        case 'n': mi_outc('\n', &out, end); break;
-        case 'r': mi_outc('\r', &out, end); break;
-        case '\\': mi_outc('\\', &out, end); break;
-        default: /* ignore */ break;
-        }
-      }
-      else if ((c >= ' ' && c <= '~') || c=='\n' || c=='\r' || c=='\t' || c=='\x1b') { // output visible ascii or standard control only
-        mi_outc(c, &out, end);
-      }
-    }
-    else {
-      MI_NEXTC();
-      char   fill = ' ';
-      size_t width = 0;
-      char   numtype = 'd';
-      char   numplus = 0;
-      bool   alignright = true;
-      if (c == '+' || c == ' ') { numplus = c; MI_NEXTC(); }
-      if (c == '-') { alignright = false; MI_NEXTC(); }
-      if (c == '0') { fill = '0'; MI_NEXTC(); }
-      if (c >= '1' && c <= '9') {
-        width = (c - '0'); MI_NEXTC();
-        while (c >= '0' && c <= '9') {
-          width = (10 * width) + (c - '0'); MI_NEXTC();
-        }
-        if (c == 0) break;  // extra check due to while
-      }
-      if (c == 'z' || c == 't' || c == 'L') { numtype = c; MI_NEXTC(); }
-      else if (c == 'l') {
-        numtype = c; MI_NEXTC();
-        if (c == 'l') { numtype = 'L'; MI_NEXTC(); }
-      }
-
-      char* start = out;
-      if (c == '%') {
-        mi_outc('%', &out, end);
-      }
-      else if (c == 's') {
-        // string
-        const char* s = va_arg(args, const char*);
-        mi_outs(s, &out, end);
-      }
-      else if (c == 'p' || c == 'x' || c == 'u') {
-        // unsigned
-        uintmax_t x = 0;
-        if (c == 'x' || c == 'u') {
-          if (numtype == 'z')       x = va_arg(args, size_t);
-          else if (numtype == 't')  x = va_arg(args, uintptr_t); // unsigned ptrdiff_t
-          else if (numtype == 'L')  x = va_arg(args, unsigned long long);
-          else if (numtype == 'l')  x = va_arg(args, unsigned long);
-                               else x = va_arg(args, unsigned int);
-        }
-        else if (c == 'p') {
-          void* const p = va_arg(args, void*);
-          x = (uintptr_t)p;
-          mi_outs("0x", &out, end);
-          start = out;
-          width = (width >= 2 ? width - 2 : 0);
-        }
-        if (width == 0 && (c == 'x' || c == 'p')) {
-          if (c == 'p')   { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); }
-          if (width == 0) { width = 2; }
-          fill = '0';
-        }
-        mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end);
-      }
-      else if (c == 'i' || c == 'd') {
-        // signed
-        intmax_t x = 0;
-        if (numtype == 'z')       x = va_arg(args, intptr_t );
-        else if (numtype == 't')  x = va_arg(args, ptrdiff_t);
-        else if (numtype == 'L')  x = va_arg(args, long long);
-        else if (numtype == 'l')  x = va_arg(args, long);
-                             else x = va_arg(args, int);
-        char pre = 0;
-        if (x < 0) {
-          pre = '-';
-          if (x > INTMAX_MIN) { x = -x; }
-        }
-        else if (numplus != 0) {
-          pre = numplus;
-        }
-        mi_out_num((uintmax_t)x, 10, pre, &out, end);
-      }
-      else if (c >= ' ' && c <= '~') {
-        // unknown format
-        mi_outc('%', &out, end);
-        mi_outc(c, &out, end);
-      }
-
-      // fill & align
-      mi_assert_internal(out <= end);
-      mi_assert_internal(out >= start);
-      const size_t len = out - start;
-      if (len < width) {
-        mi_out_fill(fill, width - len, &out, end);
-        if (alignright && out <= end) {
-          mi_out_alignright(fill, start, len, width - len, end);
-        }
-      }
-    }
-  }
-  mi_assert_internal(out <= end);
-  *out = 0;
-  return (int)(out - buf);
-}
-
-int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) {
-  va_list args;
-  va_start(args, fmt);
-  const int written = _mi_vsnprintf(buf, buflen, fmt, args);
-  va_end(args);
-  return written;
-}
-
-
-
-// --------------------------------------------------------
-// generic trailing and leading zero count, and popcount
-// --------------------------------------------------------
-
-#if !MI_HAS_FAST_BITSCAN
-
-static size_t mi_ctz_generic32(uint32_t x) {
-  // de Bruijn multiplication, see <http://keithandkatie.com/keith/papers/debruijn.html>
-  static const uint8_t debruijn[32] = {
-    0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
-    31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
-  };
-  if (x==0) return 32;
-  return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27];
-}
-
-static size_t mi_clz_generic32(uint32_t x) {
-  // de Bruijn multiplication, see <http://keithandkatie.com/keith/papers/debruijn.html>
-  static const uint8_t debruijn[32] = {
-    31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1,
-    23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0
-  };
-  if (x==0) return 32;
-  x |= x >> 1;
-  x |= x >> 2;
-  x |= x >> 4;
-  x |= x >> 8;
-  x |= x >> 16;
-  return debruijn[(uint32_t)(x * (uint32_t)(0x07C4ACDDU)) >> 27];
-}
-
-size_t _mi_ctz_generic(size_t x) {
-  if (x==0) return MI_SIZE_BITS;
-  #if (MI_SIZE_BITS <= 32)
-    return mi_ctz_generic32((uint32_t)x);
-  #else
-    const uint32_t lo = (uint32_t)x;
-    if (lo != 0) {
-      return mi_ctz_generic32(lo);
-    }
-    else {
-      return (32 + mi_ctz_generic32((uint32_t)(x>>32)));
-    }
-  #endif
-}
-
-size_t _mi_clz_generic(size_t x) {
-  if (x==0) return MI_SIZE_BITS;
-  #if (MI_SIZE_BITS <= 32)
-    return mi_clz_generic32((uint32_t)x);
-  #else
-    const uint32_t hi = (uint32_t)(x>>32);
-    if (hi != 0) {
-      return mi_clz_generic32(hi);
-    }
-    else {
-      return 32 + mi_clz_generic32((uint32_t)x);
-    }
-  #endif
-}
-
-#endif // bit scan
-
-
-#if MI_SIZE_SIZE == 4
-#define mi_mask_even_bits32      (0x55555555)
-#define mi_mask_even_pairs32     (0x33333333)
-#define mi_mask_even_nibbles32   (0x0F0F0F0F)
-
-// sum of all the bytes in `x` if it is guaranteed that the sum < 256!
-static size_t mi_byte_sum32(uint32_t x) {
-  // perform `x * 0x01010101`: the highest byte contains the sum of all bytes.
-  x += (x << 8);
-  x += (x << 16);
-  return (size_t)(x >> 24);
-}
-
-static size_t mi_popcount_generic32(uint32_t x) {
-  // first count each 2-bit group `a`, where: a==0b00 -> 00, a==0b01 -> 01, a==0b10 -> 01, a==0b11 -> 10
-  // in other words, `a - (a>>1)`; to do this in parallel, we need to mask to prevent spilling a bit pair
-  // into the lower bit-pair:
-  x = x - ((x >> 1) & mi_mask_even_bits32);
-  // add the 2-bit pair results
-  x = (x & mi_mask_even_pairs32) + ((x >> 2) & mi_mask_even_pairs32);
-  // add the 4-bit nibble results
-  x = (x + (x >> 4)) & mi_mask_even_nibbles32;
-  // each byte now has a count of its bits, we can sum them now:
-  return mi_byte_sum32(x);
-}
-
-mi_decl_noinline size_t _mi_popcount_generic(size_t x) {
-  if (x<=1) return x;
-  if (~x==0) return MI_SIZE_BITS;
-  return mi_popcount_generic32(x);
-}
-
-#else
-#define mi_mask_even_bits64      (0x5555555555555555)
-#define mi_mask_even_pairs64     (0x3333333333333333)
-#define mi_mask_even_nibbles64   (0x0F0F0F0F0F0F0F0F)
-
-// sum of all the bytes in `x` if it is guaranteed that the sum < 256!
-static size_t mi_byte_sum64(uint64_t x) {
-  x += (x << 8);
-  x += (x << 16);
-  x += (x << 32);
-  return (size_t)(x >> 56);
-}
-
-static size_t mi_popcount_generic64(uint64_t x) {
-  x = x - ((x >> 1) & mi_mask_even_bits64);
-  x = (x & mi_mask_even_pairs64) + ((x >> 2) & mi_mask_even_pairs64);
-  x = (x + (x >> 4)) & mi_mask_even_nibbles64;
-  return mi_byte_sum64(x);
-}
-
-mi_decl_noinline size_t _mi_popcount_generic(size_t x) {
-  if (x<=1) return x;
-  if (~x==0) return MI_SIZE_BITS;
-  return mi_popcount_generic64(x);
-}
-#endif
-
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/options.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/options.c
deleted file mode 100644
index 4fe7fe5a..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/options.c
+++ /dev/null
@@ -1,704 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/atomic.h"
-#include "mimalloc/prim.h"  // mi_prim_out_stderr
-
-#include <stdio.h>      // stdin/stdout
-#include <stdlib.h>     // abort
-
-static long mi_max_error_count   = 16; // stop outputting errors after this (use < 0 for no limit)
-static long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit)
-
-static void mi_add_stderr_output(void);
-
-int mi_version(void) mi_attr_noexcept {
-  return MI_MALLOC_VERSION;
-}
-
-
-// --------------------------------------------------------
-// Options
-// These can be accessed by multiple threads and may be
-// concurrently initialized, but an initializing data race
-// is ok since they resolve to the same value.
-// --------------------------------------------------------
-
-
-#define MI_OPTION(opt)                  mi_option_##opt, #opt, NULL
-#define MI_OPTION_LEGACY(opt,legacy)    mi_option_##opt, #opt, #legacy
-
-// Some options can be set at build time for statically linked libraries
-// (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`)
-//
-// This is useful if we cannot pass them as environment variables
-// (and setting them programmatically would be too late)
-
-#ifndef MI_DEFAULT_VERBOSE
-#define MI_DEFAULT_VERBOSE 0
-#endif
-
-#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT
-#define MI_DEFAULT_ARENA_EAGER_COMMIT 2
-#endif
-
-// in KiB
-#ifndef MI_DEFAULT_ARENA_RESERVE
- #if (MI_INTPTR_SIZE>4)
-  #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L
- #else
-  #define MI_DEFAULT_ARENA_RESERVE 128L*1024L
- #endif
-#endif
-
-#ifndef MI_DEFAULT_ARENA_MAX_OBJECT_SIZE
-#define MI_DEFAULT_ARENA_MAX_OBJECT_SIZE   ((MI_SIZE_BITS * MI_ARENA_MAX_CHUNK_OBJ_SIZE)/MI_KiB)  /* 2 GiB (or 256 MiB on 32-bit), larger than this is alloc'd by the OS */
-#endif
-
-#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC
-#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0
-#endif
-
-#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES
-#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0
-#endif
-
-#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES
-#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0
-#endif
-
-#ifndef MI_DEFAULT_RESERVE_OS_MEMORY
-#define MI_DEFAULT_RESERVE_OS_MEMORY 0
-#endif
-
-#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE
-#if MI_GUARDED && !MI_DEBUG
-#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000
-#else
-#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0
-#endif
-#endif
-
-#ifndef MI_DEFAULT_PAGEMAP_COMMIT
-#if defined(__APPLE__)  // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access
-#define MI_DEFAULT_PAGEMAP_COMMIT 1
-#else
-#define MI_DEFAULT_PAGEMAP_COMMIT 0
-#endif
-#endif
-
-#ifndef MI_DEFAULT_PAGE_MAX_RECLAIM
-#define MI_DEFAULT_PAGE_MAX_RECLAIM  (-1)               // unlimited
-#endif
-
-#ifndef MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM
-#define MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM  32
-#endif
-
-#ifndef MI_DEFAULT_ALLOW_THP
-#if defined(__ANDROID__)
-#define MI_DEFAULT_ALLOW_THP  0
-#else
-#define MI_DEFAULT_ALLOW_THP  1
-#endif
-#endif
-
-// Static options
-static mi_option_desc_t mi_options[_mi_option_last] =
-{
-  // stable options
-#if MI_DEBUG || defined(MI_SHOW_ERRORS)
-  { 1, MI_OPTION_UNINIT, MI_OPTION(show_errors) },
-#else
-  { 0, MI_OPTION_UNINIT, MI_OPTION(show_errors) },
-#endif
-  { 0, MI_OPTION_UNINIT, MI_OPTION(show_stats) },
-  { MI_DEFAULT_VERBOSE, MI_OPTION_UNINIT, MI_OPTION(verbose) },
-
-  // some of the following options are experimental and not all combinations are allowed.
-  { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) },  
-  { MI_DEFAULT_ARENA_EAGER_COMMIT,
-       MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux)
-  { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) },        // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit)
-  { MI_DEFAULT_ALLOW_LARGE_OS_PAGES,
-       MI_OPTION_UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) },    // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
-  { MI_DEFAULT_RESERVE_HUGE_OS_PAGES,
-       MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages) },      // per 1GiB huge pages
-  {-1, MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages_at) },   // reserve huge pages at node N
-  { MI_DEFAULT_RESERVE_OS_MEMORY,
-       MI_OPTION_UNINIT, MI_OPTION(reserve_os_memory)     },      // reserve N KiB OS memory in advance (use `option_get_size`)
-  { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_cache) },   // cache N segments per thread
-  { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_page_reset) },      // reset page memory on free
-  { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) }, 
-  { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_reset) },   // reset segment memory on free (needs eager commit)
-  { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) },  
-  { 1000,MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) },  // purge delay in milli-seconds
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(use_numa_nodes) },           // 0 = use available numa nodes, otherwise use at most N nodes.
-  { 0,   MI_OPTION_UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) },           // 1 = do not use OS memory for allocation (but only reserved arenas)
-  { 100, MI_OPTION_UNINIT, MI_OPTION(os_tag) },                   // only apple specific for now but might serve more or less related purpose
-  { 32,  MI_OPTION_UNINIT, MI_OPTION(max_errors) },               // maximum errors that are output
-  { 32,  MI_OPTION_UNINIT, MI_OPTION(max_warnings) },             // maximum warnings that are output
-  { 10,  MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)},       // max. percentage of the abandoned segments to be reclaimed per try.
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)},           // release all OS memory on process exit; careful with dangling pointer or after-exit frees!
-  { MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)
-  { 1,   MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) },         // purge delay multiplier for arena's
-  { 1,   MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) },
-  { MI_DEFAULT_DISALLOW_ARENA_ALLOC,   MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)
-  { 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) },             // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.
-#if defined(MI_VISIT_ABANDONED)
-  { 1,   MI_OPTION_INITIALIZED, MI_OPTION(visit_abandoned) },     // allow visiting theap blocks in abandoned segments; requires taking locks during reclaim.
-#else
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(visit_abandoned) },
-#endif
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(guarded_min) },              // only used when building with MI_GUARDED: minimal rounded object size for guarded objects
-  { MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) },           // only used when building with MI_GUARDED: maximal rounded object size for guarded objects
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(guarded_precise) },          // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
-  { MI_DEFAULT_GUARDED_SAMPLE_RATE,
-         MI_OPTION_UNINIT, MI_OPTION(guarded_sample_rate)},       // 1 out of N allocations in the min/max range will be guarded (=4000)
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(guarded_sample_seed)},
-  { 10000, MI_OPTION_UNINIT, MI_OPTION(generic_collect) },        // collect theaps every N (=10000) generic allocation calls
-  { 0,   MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 = only reclaim into the originating theap, 1 = reclaim on free across theaps
-  { 2,   MI_OPTION_UNINIT, MI_OPTION(page_full_retain) },         // number of (small) pages to retain in the free page queues
-  { 4,   MI_OPTION_UNINIT, MI_OPTION(page_max_candidates) },      // max search to find a best page candidate
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(max_vabits) },               // max virtual address space bits
-  { MI_DEFAULT_PAGEMAP_COMMIT,
-         MI_OPTION_UNINIT, MI_OPTION(pagemap_commit) },           // commit the full pagemap upfront?
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(page_commit_on_demand) },    // commit pages on-demand (2 disables this only on overcommit systems (like Linux))
-  { MI_DEFAULT_PAGE_MAX_RECLAIM,
-         MI_OPTION_UNINIT, MI_OPTION(page_max_reclaim) },         // don't reclaim (small) pages of the same originating theap if we already own N pages in that size class
-  { MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM,
-         MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class
-  { MI_DEFAULT_ALLOW_THP,
-         MI_OPTION_UNINIT, MI_OPTION(allow_thp) },                // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) },       // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`).
-  { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE,   
-         MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) },    // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). 
-  { 0,   MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) },      // associate local numa node with an initial arena allocation
-};
-
-static void mi_option_init(mi_option_desc_t* desc);
-
-static bool mi_option_has_size_in_kib(mi_option_t option) {
-  return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve || 
-          option == mi_option_minimal_purge_size || option == mi_option_arena_max_object_size);
-}
-
-void _mi_options_init(void) {
-  // called on process load
-  for(int i = 0; i < _mi_option_last; i++ ) {
-    mi_option_t option = (mi_option_t)i;
-    long l = mi_option_get(option); MI_UNUSED(l); // initialize
-  }
-  mi_max_error_count = mi_option_get(mi_option_max_errors);
-  mi_max_warning_count = mi_option_get(mi_option_max_warnings);
-  #if MI_GUARDED
-  if (mi_option_get(mi_option_guarded_sample_rate) > 0) {
-    if (mi_option_is_enabled(mi_option_allow_large_os_pages)) {
-      mi_option_disable(mi_option_allow_large_os_pages);
-      _mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n");
-    }
-  }
-  #endif  
-}
-
-// called at actual process load, it should be safe to print now
-void _mi_options_post_init(void) {
-  mi_add_stderr_output(); // now it safe to use stderr for output
-  if (mi_option_is_enabled(mi_option_verbose)) { mi_options_print(); }
-}
-
-#define mi_stringifyx(str)  #str                // and stringify
-#define mi_stringify(str)   mi_stringifyx(str)  // expand
-
-mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept
-{
-  // show version
-  const int vermajor = MI_MALLOC_VERSION/10000;
-  const int verminor = (MI_MALLOC_VERSION%10000)/100;
-  const int verpatch = (MI_MALLOC_VERSION%100);
-  _mi_fprintf(out, arg, "v%i.%i.%i%s%s (built on %s, %s)\n", vermajor, verminor, verpatch,
-      #if defined(MI_CMAKE_BUILD_TYPE)
-      ", " mi_stringify(MI_CMAKE_BUILD_TYPE)
-      #else
-      ""
-      #endif
-      ,
-      #if defined(MI_GIT_DESCRIBE)
-      ", git " mi_stringify(MI_GIT_DESCRIBE)
-      #else
-      ""
-      #endif
-      , __DATE__, __TIME__);
-
-  // show options
-  for (int i = 0; i < _mi_option_last; i++) {
-    mi_option_t option = (mi_option_t)i;
-    long l = mi_option_get(option); MI_UNUSED(l); // possibly initialize
-    mi_option_desc_t* desc = &mi_options[option];
-    _mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : ""));
-  }
-
-  // show build configuration
-  _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG );
-  _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE );
-  _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL);
-  #if MI_GUARDED
-  _mi_fprintf(out, arg, "guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled");
-  #endif
-  #if MI_TSAN
-  _mi_fprintf(out, arg, "thread santizer enabled\n");
-  #endif
-}
-
-mi_decl_export void mi_options_print(void) mi_attr_noexcept {
-  mi_options_print_out(NULL, NULL);
-}
-
-long _mi_option_get_fast(mi_option_t option) {
-  mi_assert(option >= 0 && option < _mi_option_last);
-  mi_option_desc_t* desc = &mi_options[option];
-  mi_assert(desc->option == option);  // index should match the option
-  //mi_assert(desc->init != MI_OPTION_UNINIT);
-  return desc->value;
-}
-
-
-mi_decl_nodiscard long mi_option_get(mi_option_t option) {
-  mi_assert(option >= 0 && option < _mi_option_last);
-  if (option < 0 || option >= _mi_option_last) return 0;
-  mi_option_desc_t* desc = &mi_options[option];
-  mi_assert(desc->option == option);  // index should match the option
-  if mi_unlikely(desc->init == MI_OPTION_UNINIT) {
-    mi_option_init(desc);
-  }
-  return desc->value;
-}
-
-mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long max) {
-  long x = mi_option_get(option);
-  return (x < min ? min : (x > max ? max : x));
-}
-
-mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {
-  const long x = mi_option_get(option);
-  size_t size = (x < 0 ? 0 : (size_t)x);
-  if (mi_option_has_size_in_kib(option)) {
-    if (mi_mul_overflow(size, MI_KiB, &size)) {
-      size = MI_MAX_ALLOC_SIZE;
-    }
-  }
-  return size;
-}
-
-void mi_option_set(mi_option_t option, long value) {
-  mi_assert(option >= 0 && option < _mi_option_last);
-  if (option < 0 || option >= _mi_option_last) return;
-  mi_option_desc_t* desc = &mi_options[option];
-  mi_assert(desc->option == option);  // index should match the option
-  desc->value = value;
-  desc->init = MI_OPTION_INITIALIZED;
-  // ensure min/max range; be careful to not recurse.
-  if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) {
-    mi_option_set(mi_option_guarded_max, value);
-  }
-  else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) {
-    mi_option_set(mi_option_guarded_min, value);
-  }
-}
-
-void mi_option_set_default(mi_option_t option, long value) {
-  mi_assert(option >= 0 && option < _mi_option_last);
-  if (option < 0 || option >= _mi_option_last) return;
-  mi_option_desc_t* desc = &mi_options[option];
-  if (desc->init != MI_OPTION_INITIALIZED) {
-    desc->value = value;
-  }
-}
-
-mi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option) {
-  return (mi_option_get(option) != 0);
-}
-
-void mi_option_set_enabled(mi_option_t option, bool enable) {
-  mi_option_set(option, (enable ? 1 : 0));
-}
-
-void mi_option_set_enabled_default(mi_option_t option, bool enable) {
-  mi_option_set_default(option, (enable ? 1 : 0));
-}
-
-void mi_option_enable(mi_option_t option) {
-  mi_option_set_enabled(option,true);
-}
-
-void mi_option_disable(mi_option_t option) {
-  mi_option_set_enabled(option,false);
-}
-
-static void mi_cdecl mi_out_stderr(const char* msg, void* arg) {
-  MI_UNUSED(arg);
-  if (msg != NULL && msg[0] != 0) {
-    _mi_prim_out_stderr(msg);
-  }
-}
-
-// Since an output function can be registered earliest in the `main`
-// function we also buffer output that happens earlier. When
-// an output function is registered it is called immediately with
-// the output up to that point.
-#ifndef MI_MAX_DELAY_OUTPUT
-#define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024))
-#endif
-static char out_buf[MI_MAX_DELAY_OUTPUT+1];
-static _Atomic(size_t) out_len;
-static mi_lock_t out_buf_lock = MI_LOCK_INITIALIZER;
-
-static void mi_cdecl mi_out_buf(const char* msg, void* arg) {
-  MI_UNUSED(arg);
-  if (msg==NULL) return;
-  if (mi_atomic_load_acquire(&out_len)>=MI_MAX_DELAY_OUTPUT) return;
-  size_t n = _mi_strlen(msg);
-  if (n==0 || n >= MI_MAX_DELAY_OUTPUT) return;
-  // copy msg into the buffer
-  mi_lock(&out_buf_lock) {
-    const size_t start = mi_atomic_add_acq_rel(&out_len, n);
-    if (start < MI_MAX_DELAY_OUTPUT) {
-      // check bound
-      if (start+n >= MI_MAX_DELAY_OUTPUT) {
-        n = MI_MAX_DELAY_OUTPUT-start-1;
-      }
-      _mi_memcpy(&out_buf[start], msg, n);
-    }
-  }
-}
-
-static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) {
-  if (out==NULL) return;
-  // claim (if `no_more_buf == true`, no more output will be added after this point)
-  mi_lock(&out_buf_lock) {
-    size_t count = mi_atomic_add_acq_rel(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1));
-    // and output the current contents
-    if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT;
-    out_buf[count] = 0;
-    out(out_buf,arg);
-    if (!no_more_buf) {
-      out_buf[count] = '\n'; // if continue with the buffer, insert a newline
-    }
-  }
-}
-
-
-// Once this module is loaded, switch to this routine
-// which outputs to stderr and the delayed output buffer.
-static void mi_cdecl mi_out_buf_stderr(const char* msg, void* arg) {
-  mi_out_stderr(msg,arg);
-  mi_out_buf(msg,arg);
-}
-
-
-
-// --------------------------------------------------------
-// Default output handler
-// --------------------------------------------------------
-
-// The program should only install a single output handler from a single thread
-// since otherwise the argument and output function may not match.
-static _Atomic(void*) mi_out_default; // = // is `mi_output_fun*` (but some platforms don't support atomic function pointers)
-static _Atomic(void*) mi_out_arg; // = NULL
-
-static mi_output_fun* mi_out_get_default(void** parg) {
-  mi_output_fun* const out = (mi_output_fun*)mi_atomic_load_ptr_acquire(void,&mi_out_default);
-  if (parg != NULL) { *parg = mi_atomic_load_ptr_acquire(void,&mi_out_arg); }
-  return (out == NULL ? &mi_out_buf : out);
-}
-
-void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept {
-  mi_atomic_store_ptr_release(void,&mi_out_default, (void*)(out == NULL ? &mi_out_stderr : out)); // stop using the delayed output buffer
-  mi_atomic_store_ptr_release(void,&mi_out_arg, arg);
-  if (out!=NULL) { mi_out_buf_flush(out,true,arg); }        // output all the delayed output now
-}
-
-// add stderr to the delayed output after the module is loaded
-static void mi_add_stderr_output(void) {
-  mi_assert_internal(mi_out_default == NULL);
-  mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr
-  mi_atomic_store_ptr_release(void,&mi_out_default,(void*)&mi_out_buf_stderr);  // and add stderr to the delayed output
-  mi_atomic_store_ptr_release(void,&mi_out_arg,NULL);
-}
-
-// --------------------------------------------------------
-// Messages, all end up calling `_mi_fputs`.
-// --------------------------------------------------------
-static _Atomic(size_t) error_count;   // = 0;  // when >= max_error_count stop emitting errors
-static _Atomic(size_t) warning_count; // = 0;  // when >= max_warning_count stop emitting warnings
-
-// When overriding malloc, we may recurse into mi_vfprintf if an allocation
-// inside the C runtime causes another message.
-// In some cases (like on macOS) the loader already allocates which
-// calls into mimalloc; if we then access thread locals (like `recurse`)
-// this may crash as the access may call _tlv_bootstrap that tries to
-// (recursively) invoke malloc again to allocate space for the thread local
-// variables on demand. This is why we use a _mi_preloading test on such
-// platforms. However, C code generator may move the initial thread local address
-// load before the `if` and we therefore split it out in a separate function.
-static mi_decl_thread bool recurse = false;
-
-static mi_decl_noinline bool mi_recurse_enter_prim(void) {
-  if (recurse) return false;
-  recurse = true;
-  return true;
-}
-
-static mi_decl_noinline void mi_recurse_exit_prim(void) {
-  recurse = false;
-}
-
-static bool mi_recurse_enter(void) {
-  #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)
-  if (_mi_preloading()) return false;
-  #endif
-  return mi_recurse_enter_prim();
-}
-
-static void mi_recurse_exit(void) {
-  #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)
-  if (_mi_preloading()) return;
-  #endif
-  mi_recurse_exit_prim();
-}
-
-void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {
-  if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // TODO: use mi_out_stderr for stderr?
-    if (!mi_recurse_enter()) return;
-    out = mi_out_get_default(&arg);
-    if (prefix != NULL) out(prefix, arg);
-    out(message, arg);
-    mi_recurse_exit();
-  }
-  else {
-    if (prefix != NULL) out(prefix, arg);
-    out(message, arg);
-  }
-}
-
-// Define our own limited `fprintf` that avoids memory allocation.
-// We do this using `_mi_vsnprintf` with a limited buffer.
-static void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) {
-  char buf[992];
-  if (fmt==NULL) return;
-  if (!mi_recurse_enter()) return;
-  _mi_vsnprintf(buf, sizeof(buf)-1, fmt, args);
-  mi_recurse_exit();
-  _mi_fputs(out,arg,prefix,buf);
-}
-
-void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {
-  va_list args;
-  va_start(args,fmt);
-  mi_vfprintf(out,arg,NULL,fmt,args);
-  va_end(args);
-}
-
-static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {
-  if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) {
-    char tprefix[64];
-    _mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id());
-    mi_vfprintf(out, arg, tprefix, fmt, args);
-  }
-  else {
-    mi_vfprintf(out, arg, prefix, fmt, args);
-  }
-}
-
-void _mi_raw_message(const char* fmt, ...) {
-  va_list args;
-  va_start(args, fmt);
-  mi_vfprintf(NULL, NULL, NULL, fmt, args);
-  va_end(args);
-}
-
-void _mi_message(const char* fmt, ...) {
-  va_list args;
-  va_start(args, fmt);
-  mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args);
-  va_end(args);
-}
-
-void _mi_trace_message(const char* fmt, ...) {
-  if (mi_option_get(mi_option_verbose) <= 1) return;  // only with verbose level 2 or higher
-  va_list args;
-  va_start(args, fmt);
-  mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args);
-  va_end(args);
-}
-
-void _mi_verbose_message(const char* fmt, ...) {
-  if (!mi_option_is_enabled(mi_option_verbose)) return;
-  va_list args;
-  va_start(args,fmt);
-  mi_vfprintf(NULL, NULL, "mimalloc: ", fmt, args);
-  va_end(args);
-}
-
-static void mi_show_error_message(const char* fmt, va_list args) {
-  if (!mi_option_is_enabled(mi_option_verbose)) {
-    if (!mi_option_is_enabled(mi_option_show_errors)) return;
-    if (mi_max_error_count >= 0 && (long)mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return;
-  }
-  mi_vfprintf_thread(NULL, NULL, "mimalloc: error: ", fmt, args);
-}
-
-void _mi_warning_message(const char* fmt, ...) {
-  if (!mi_option_is_enabled(mi_option_verbose)) {
-    if (!mi_option_is_enabled(mi_option_show_errors)) return;
-    if (mi_max_warning_count >= 0 && (long)mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return;
-  }
-  va_list args;
-  va_start(args,fmt);
-  mi_vfprintf_thread(NULL, NULL, "mimalloc: warning: ", fmt, args);
-  va_end(args);
-}
-
-
-#if MI_DEBUG
-mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept {
-  _mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n  assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion);
-  abort();
-}
-#endif
-
-// --------------------------------------------------------
-// Errors
-// --------------------------------------------------------
-
-static mi_error_fun* volatile  mi_error_handler; // = NULL
-static _Atomic(void*) mi_error_arg;     // = NULL
-
-static void mi_error_default(int err) {
-  MI_UNUSED(err);
-#if (MI_DEBUG>0)
-  if (err==EFAULT) {
-    #ifdef _MSC_VER
-    __debugbreak();
-    #endif
-    abort();
-  }
-#endif
-#if (MI_SECURE>0)
-  if (err==EFAULT) {  // abort on serious errors in secure mode (corrupted meta-data)
-    abort();
-  }
-#endif
-#if defined(MI_XMALLOC)
-  if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode
-    abort();
-  }
-#endif
-}
-
-void mi_register_error(mi_error_fun* fun, void* arg) {
-  mi_error_handler = fun;  // can be NULL
-  mi_atomic_store_ptr_release(void,&mi_error_arg, arg);
-}
-
-void _mi_error_message(int err, const char* fmt, ...) {
-  // show detailed error message
-  va_list args;
-  va_start(args, fmt);
-  mi_show_error_message(fmt, args);
-  va_end(args);
-  // and call the error handler which may abort (or return normally)
-  if (mi_error_handler != NULL) {
-    mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg));
-  }
-  else {
-    mi_error_default(err);
-  }
-}
-
-// --------------------------------------------------------
-// Initialize options by checking the environment
-// --------------------------------------------------------
-
-// TODO: implement ourselves to reduce dependencies on the C runtime
-#include <stdlib.h> // strtol
-#include <string.h> // strstr
-
-
-static void mi_option_init(mi_option_desc_t* desc) {
-  // Read option value from the environment
-  char s[64 + 1];
-  char buf[64+1];
-  _mi_strlcpy(buf, "mimalloc_", sizeof(buf));
-  _mi_strlcat(buf, desc->name, sizeof(buf));
-  bool found = _mi_getenv(buf, s, sizeof(s));
-  if (!found && desc->legacy_name != NULL) {
-    _mi_strlcpy(buf, "mimalloc_", sizeof(buf));
-    _mi_strlcat(buf, desc->legacy_name, sizeof(buf));
-    found = _mi_getenv(buf, s, sizeof(s));
-    if (found) {
-      _mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name);
-    }
-  }
-
-  if (found) {
-    size_t len = _mi_strnlen(s, sizeof(buf) - 1);
-    for (size_t i = 0; i < len; i++) {
-      buf[i] = _mi_toupper(s[i]);
-    }
-    buf[len] = 0;
-    if (buf[0] == 0 || _mi_streq(buf,"1") || _mi_streq(buf,"TRUE") || _mi_streq(buf,"YES") || _mi_streq(buf,"ON")) {
-      desc->value = 1;
-      desc->init = MI_OPTION_INITIALIZED;
-    }
-    else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) {      
-      desc->value = 0;
-      desc->init = MI_OPTION_INITIALIZED;
-    }
-    else {
-      char* end = buf;
-      long value = strtol(buf, &end, 10);
-      if (mi_option_has_size_in_kib(desc->option)) {
-        // this option is interpreted in KiB to prevent overflow of `long` for large allocations
-        // (long is 32-bit on 64-bit windows, which allows for 4TiB max.)
-        size_t size = (value < 0 ? 0 : (size_t)value);
-        bool overflow = false;
-        if (*end == 'K') { end++; }
-        else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; }
-        else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; }
-        else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; }
-        else { size = (size + MI_KiB - 1) / MI_KiB; }
-        if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB
-        else if (*end == 'B') { end++; }                  // Kb, Mb, Gb, Tb
-        if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); }
-        value = (size > LONG_MAX ? LONG_MAX : (long)size);
-      }
-      if (*end == 0) {
-        mi_option_set(desc->option, value);
-      }
-      else {
-        // set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose.
-        desc->init = MI_OPTION_DEFAULTED;
-        if (desc->option == mi_option_verbose && desc->value == 0) {
-          // if the 'mimalloc_verbose' env var has a bogus value we'd never know
-          // (since the value defaults to 'off') so in that case briefly enable verbose
-          desc->value = 1;
-          _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name);
-          desc->value = 0;
-        }
-        else {
-          _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name);
-        }
-      }
-    }
-    mi_assert_internal(desc->init != MI_OPTION_UNINIT);
-  }
-  else if (!_mi_preloading()) {
-    desc->init = MI_OPTION_DEFAULTED;
-  }
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/os.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/os.c
deleted file mode 100644
index febecb0c..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/os.c
+++ /dev/null
@@ -1,923 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/atomic.h"
-#include "mimalloc/prim.h"
-
-/* -----------------------------------------------------------
-  Initialization.
------------------------------------------------------------ */
-#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB
-#if MI_INTPTR_SIZE < 8
-#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB   4*MI_MiB    // 4 GiB
-#else
-#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB   32*MI_MiB   // 32 GiB
-#endif
-#endif
-
-static mi_os_mem_config_t mi_os_mem_config = {
-  4096,     // page size
-  0,        // large page size (usually 2MiB)
-  4096,     // allocation granularity
-  MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB,
-  MI_MAX_VABITS, // in `bits.h`
-  true,     // has overcommit?  (if true we use MAP_NORESERVE on mmap systems)
-  false,    // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
-  true,     // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)
-  false     // has transparent huge pages? (if true we purge in (aligned) large page size chunks only to not fragment such pages)
-};
-
-bool _mi_os_has_overcommit(void) {
-  return mi_os_mem_config.has_overcommit;
-}
-
-bool _mi_os_has_virtual_reserve(void) {
-  return mi_os_mem_config.has_virtual_reserve;
-}
-
-
-// OS (small) page size
-size_t _mi_os_page_size(void) {
-  return mi_os_mem_config.page_size;
-}
-
-// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB)
-size_t _mi_os_large_page_size(void) {
-  return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size());
-}
-
-// minimal purge size. Can be larger than the page size if transparent huge pages are enabled.
-size_t _mi_os_minimal_purge_size(void) {
-  size_t minsize = mi_option_get_size(mi_option_minimal_purge_size);
-  if (minsize != 0) {
-    return _mi_align_up(minsize, _mi_os_page_size());
-  }
-  else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) {
-    return _mi_os_large_page_size();
-  }
-  else {
-    return _mi_os_page_size();
-  }
-}
-
-size_t _mi_os_guard_page_size(void) {
-  const size_t gsize = _mi_os_page_size();
-  mi_assert(gsize <= (MI_ARENA_SLICE_SIZE/4)); // issue #1166
-  return gsize;
-}
-
-size_t _mi_os_virtual_address_bits(void) {
-  const size_t vbits = mi_os_mem_config.virtual_address_bits;
-  mi_assert(vbits <= MI_MAX_VABITS);
-  return vbits;
-}
-
-bool _mi_os_canuse_large_page(size_t size, size_t alignment) {
-  // if we have access, check the size and alignment requirements
-  if (mi_os_mem_config.large_page_size == 0) return false;
-  return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0);
-}
-
-// round to a good OS allocation size (bounded by max 12.5% waste)
-size_t _mi_os_good_alloc_size(size_t size) {
-  size_t align_size;
-  if (size < 512*MI_KiB) align_size = _mi_os_page_size();
-  else if (size < 2*MI_MiB) align_size = 64*MI_KiB;
-  else if (size < 8*MI_MiB) align_size = 256*MI_KiB;
-  else if (size < 32*MI_MiB) align_size = 1*MI_MiB;
-  else align_size = 4*MI_MiB;
-  if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow?
-  return _mi_align_up(size, align_size);
-}
-
-void _mi_os_init(void) {
-  _mi_prim_mem_init(&mi_os_mem_config);
-}
-
-
-/* -----------------------------------------------------------
-  Util
--------------------------------------------------------------- */
-bool _mi_os_decommit(void* addr, size_t size);
-bool _mi_os_commit(void* addr, size_t size, bool* is_zero);
-
-// On systems with enough virtual address bits, we can do efficient aligned allocation by using
-// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address
-// space (64TiB) we use this technique. (but see issue #939)
-#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID)
-
-// Return a MI_HINT_ALIGN (4MiB) aligned address that is probably available.
-// If this returns NULL, the OS will determine the address but on some OS's that may not be
-// properly aligned which can be more costly as it needs to be adjusted afterwards.
-// For a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization;
-// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses
-//  in the middle of the 2TiB - 6TiB address range (see issue #372))
-
-#define MI_HINT_ALIGN ((uintptr_t)4 << 20)  // 4MiB alignment
-#define MI_HINT_BASE  ((uintptr_t)2 << 40)  // 2TiB start
-#define MI_HINT_AREA  ((uintptr_t)4 << 40)  // upto (2+4) 6TiB  (since before win8 there is "only" 8TiB available to processes)
-#define MI_HINT_MAX   ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)
-
-void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
-{
-  static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0
-
-  // todo: perhaps only do alignment hints if THP is enabled?
-  if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL;
-  if (mi_os_mem_config.virtual_address_bits < 46) return NULL;  // < 64TiB virtual address space
-  size = _mi_align_up(size, MI_HINT_ALIGN);
-  if (size > 16*MI_GiB) return NULL;  // guarantee the chance of fixed valid address is at least 1/(MI_HINT_AREA / 1<<34)
-  size += MI_HINT_ALIGN;              // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.
-
-  uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);
-  if (hint == 0 || hint > MI_HINT_MAX) {   // wrap or initialize
-    uintptr_t init = MI_HINT_BASE;
-    #if (MI_SECURE>=1 || defined(NDEBUG))  // security: randomize start of aligned allocations unless in debug mode
-    mi_theap_t* const theap = _mi_theap_default();     // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
-    if (!mi_theap_is_initialized(theap)) return NULL;  // no hint as we lack randomness at this point
-    const uintptr_t r = _mi_theap_random_next(theap);
-    init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA);  // (randomly 20 bits)*4MiB == 0 to 4TiB
-    #endif
-    uintptr_t expected = hint + size;
-    mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
-    hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all
-  }
-  mi_assert_internal(hint%MI_HINT_ALIGN == 0);
-  if (hint%try_alignment != 0) return NULL;
-  return (void*)hint;
-}
-#else
-void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
-   MI_UNUSED(try_alignment); MI_UNUSED(size);
-   return NULL;
-}
-#endif
-
-
-/* -----------------------------------------------------------
-  Guard page allocation
------------------------------------------------------------ */
-
-// In secure mode, return the size of a guard page, otherwise 0
-size_t _mi_os_secure_guard_page_size(void) {
-  #if MI_SECURE > 0
-  return _mi_os_guard_page_size();
-  #else
-  return 0;
-  #endif
-}
-
-// In secure mode, try to decommit an area and output a warning if this fails.
-bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {
-  if (addr == NULL) return true;
-  #if MI_SECURE > 0
-  bool ok = false;
-  if (!memid.is_pinned) {
-    mi_arena_t* const arena = mi_memid_arena(memid);
-    if (arena != NULL && arena->commit_fun != NULL) {
-      ok = (*(arena->commit_fun))(false /* decommit */, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);
-    }
-    else {
-      ok = _mi_os_decommit(addr, _mi_os_secure_guard_page_size());
-    }
-  }
-  if (!ok) {
-    _mi_error_message(EINVAL, "secure level %d, but failed to commit guard page (at %p of size %zu)\n", MI_SECURE, addr, _mi_os_secure_guard_page_size());
-  }
-  return ok;
-  #else
-  MI_UNUSED(memid);
-  return true;
-  #endif
-}
-
-// In secure mode, try to decommit an area and output a warning if this fails.
-bool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid) {
-  return _mi_os_secure_guard_page_set_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
-}
-
-// In secure mode, try to recommit an area
-bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) {
-  if (addr == NULL) return true;
-  #if MI_SECURE > 0
-  if (!memid.is_pinned) {
-    mi_arena_t* const arena = mi_memid_arena(memid);
-    if (arena != NULL && arena->commit_fun != NULL) {
-      return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);
-    }
-    else {
-      return _mi_os_commit(addr, _mi_os_secure_guard_page_size(), NULL);
-    }
-  }
-  #else
-  MI_UNUSED(memid);
-  #endif
-  return true;
-}
-
-// In secure mode, try to recommit an area
-bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) {
-  return _mi_os_secure_guard_page_reset_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
-}
-
-
-/* -----------------------------------------------------------
-  Free memory
--------------------------------------------------------------- */
-
-static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc);
-
-static void mi_os_prim_free(void* addr, size_t size, size_t commit_size, mi_subproc_t* subproc) {
-  mi_assert_internal((size % _mi_os_page_size()) == 0);
-  if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr)
-  int err = _mi_prim_free(addr, size);  // allow size==0 (issue #1041)
-  if (err != 0) {
-    _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
-  }
-  if (subproc == NULL) { subproc = _mi_subproc(); } // from `mi_arenas_unsafe_destroy` we pass subproc_main explicitly as we can no longer use the theap pointer
-  if (commit_size > 0) {
-    mi_subproc_stat_decrease(subproc, committed, commit_size);
-  }
-  mi_subproc_stat_decrease(subproc, reserved, size);
-}
-
-void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc /* can be NULL */) {
-  if (mi_memkind_is_os(memid.memkind)) {
-    size_t csize = memid.mem.os.size;
-    if (csize==0) { csize = _mi_os_good_alloc_size(size); }
-    mi_assert_internal(csize >= size);
-    size_t commit_size = (still_committed ? csize : 0);
-    void* base = addr;
-    // different base? (due to alignment)
-    if (memid.mem.os.base != base) {
-      mi_assert(memid.mem.os.base <= addr);
-      base = memid.mem.os.base;
-      const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base;
-      if (memid.mem.os.size==0) {
-        csize += diff;
-      }
-      if (still_committed) {
-        commit_size -= diff;  // the (addr-base) part was already un-committed
-      }
-    }
-    // free it
-    if (memid.memkind == MI_MEM_OS_HUGE) {
-      mi_assert(memid.is_pinned);
-      mi_os_free_huge_os_pages(base, csize, subproc);
-    }
-    else {
-      mi_os_prim_free(base, csize, (still_committed ? commit_size : 0), subproc);
-    }
-  }
-  else {
-    // nothing to do
-    mi_assert(memid.memkind < MI_MEM_OS);
-  }
-}
-
-void  _mi_os_free(void* p, size_t size, mi_memid_t memid) {
-  _mi_os_free_ex(p, size, true, memid, NULL);
-}
-
-
-/* -----------------------------------------------------------
-   Primitive allocation from the OS.
--------------------------------------------------------------- */
-
-// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
-// Also `hint_addr` is a hint and may be ignored.
-static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
-  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
-  mi_assert_internal(is_zero != NULL);
-  mi_assert_internal(is_large != NULL);
-  if (size == 0) return NULL;
-  if (!commit) { allow_large = false; }
-  if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning
-
-  // try to align along large OS page size for larger allocations
-  const size_t large_page_size = mi_os_mem_config.large_page_size;
-  if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && _mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) {
-    try_alignment = large_page_size;
-  }
-
-  *is_zero = false;
-  void* p = NULL;
-  int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p);
-  if (err != 0) {
-    _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);
-  }
-
-  mi_os_stat_counter_increase(mmap_calls, 1);
-  if (p != NULL) {
-    mi_os_stat_increase(reserved, size);
-    if (commit) {
-      mi_os_stat_increase(committed, size);
-      // seems needed for asan (or `mimalloc-test-api` fails)
-      #ifdef MI_TRACK_ASAN
-      if (*is_zero) { mi_track_mem_defined(p,size); }
-               else { mi_track_mem_undefined(p,size); }
-      #endif
-    }
-  }
-  return p;
-}
-
-static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
-  return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero);
-}
-
-
-// Primitive aligned allocation from the OS.
-// This function guarantees the allocated memory is aligned.
-static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {
-  mi_assert_internal(memid!=NULL);
-  mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
-  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
-  *memid = _mi_memid_none();
-  if (!commit) allow_large = false;
-  if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
-  size = _mi_align_up(size, _mi_os_page_size());
-
-  // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size.
-  const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4);
-
-  bool os_is_large = false;
-  bool os_is_zero = false;
-  void*  os_base = NULL;
-  size_t os_size = size;
-  void* p = NULL;
-  if (try_direct_alloc) {
-    p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero);
-  }
-
-  // aligned already?
-  if (p != NULL && _mi_is_aligned(p,alignment)) {
-    os_base = p;
-  }
-  else {
-    // if not aligned, free it, overallocate, and unmap around it
-    #if !MI_TRACK_ASAN
-    if (try_direct_alloc) {
-      _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
-    }
-    #endif
-    if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0), NULL); }
-    if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
-    const size_t over_size = size + alignment;
-
-    if (!mi_os_mem_config.has_partial_free) {  // win32 virtualAlloc cannot free parts of an allocated block
-      // over-allocate uncommitted (virtual) memory
-      p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);
-      if (p == NULL) return NULL;
-
-      // set p to the aligned part in the full region
-      // note: Windows VirtualFree needs the actual base pointer
-      // this is handled though by having the `base` field in the memid
-      os_base = p; // remember the base
-      os_size = over_size;
-      p = _mi_align_up_ptr(p, alignment);
-
-      // explicitly commit only the aligned part
-      if (commit) {
-        if (!_mi_os_commit(p, size, NULL)) {
-          mi_os_prim_free(os_base, over_size, 0, NULL);
-          return NULL;
-        }
-      }
-    }
-    else  { // mmap can free inside an allocation
-      // overallocate...
-      p = mi_os_prim_alloc(over_size, 1, commit, false, &os_is_large, &os_is_zero);
-      if (p == NULL) return NULL;
-
-      // and selectively unmap parts around the over-allocated area.
-      void* const aligned_p = _mi_align_up_ptr(p, alignment);
-      const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
-      const size_t mid_size = _mi_align_up(size, _mi_os_page_size());
-      const size_t post_size = over_size - pre_size - mid_size;
-      mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
-      if (pre_size > 0)  { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0), NULL); }
-      if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), NULL); }
-      // we can return the aligned pointer on `mmap` systems
-      p = aligned_p;
-      os_base = aligned_p; // since we freed the pre part, `*base == p`.
-      os_size = mid_size;
-    }
-  }
-
-  mi_assert_internal(p != NULL && os_base != NULL && _mi_is_aligned(p,alignment));
-  mi_assert_internal(os_base <= p && size <= os_size);
-  *memid = _mi_memid_create_os(os_base,os_size,commit,os_is_zero,os_is_large);
-  return p;
-}
-
-
-/* -----------------------------------------------------------
-  OS API: alloc and alloc_aligned
------------------------------------------------------------ */
-
-void* _mi_os_alloc(size_t size, mi_memid_t* memid) {
-  *memid = _mi_memid_none();
-  if (size == 0) return NULL;
-  size = _mi_os_good_alloc_size(size);
-  bool os_is_large = false;
-  bool os_is_zero  = false;
-  void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero);
-  if (p == NULL) return NULL;
-
-  *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large);
-  mi_assert_internal(memid->mem.os.size >= size);
-  mi_assert_internal(memid->initially_committed);
-  return p;
-}
-
-void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)
-{
-  MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings
-  *memid = _mi_memid_none();
-  if (size == 0) return NULL;
-  size = _mi_os_good_alloc_size(size);
-  alignment = _mi_align_up(alignment, _mi_os_page_size());
-
-  void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid );
-  if (p == NULL) return NULL;
-
-  mi_assert_internal(memid->mem.os.size >= size);
-  mi_assert_internal(_mi_is_aligned(p,alignment));
-  if (commit) { mi_assert_internal(memid->initially_committed); }
-  return p;
-}
-
-
-mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) {
-  if (p==NULL || size==0) return p;
-  // ensure committed
-  if (!memid->initially_committed) {
-    bool is_zero = false;
-    if (!_mi_os_commit(p, size, &is_zero)) {
-      _mi_os_free(p, size, *memid);
-      return NULL;
-    }
-    memid->initially_committed = true;
-  }
-  // ensure zero'd
-  if (memid->initially_zero) return p;
-  _mi_memzero_aligned(p,size);
-  memid->initially_zero = true;
-  return p;
-}
-
-void*  _mi_os_zalloc(size_t size, mi_memid_t* memid) {
-  void* p = _mi_os_alloc(size,memid);
-  return mi_os_ensure_zero(p, size, memid);
-}
-
-/* -----------------------------------------------------------
-  OS aligned allocation with an offset. This is used
-  for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc
-  page where the object can be aligned at an offset from the start of the segment.
-  As we may need to overallocate, we need to free such pointers using `mi_free_aligned`
-  to use the actual start of the memory region.
------------------------------------------------------------ */
-
-void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {
-  mi_assert(offset <= size);
-  mi_assert((alignment % _mi_os_page_size()) == 0);
-  *memid = _mi_memid_none();
-  if (offset > size) return NULL;
-  if (offset == 0) {
-    // regular aligned allocation
-    return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);
-  }
-  else {
-    // overallocate to align at an offset
-    const size_t extra = _mi_align_up(offset, alignment) - offset;
-    if (size >= SIZE_MAX - extra) return NULL;  // too large
-    const size_t oversize = size + extra;
-    void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid);
-    if (start == NULL) return NULL;
-
-    void* const p = (uint8_t*)start + extra;
-    mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));
-    // decommit the overallocation at the start
-    if (commit && extra >= _mi_os_page_size()) {
-      _mi_os_decommit(start, extra);
-    }
-    return p;
-  }
-}
-
-/* -----------------------------------------------------------
-  OS memory API: reset, commit, decommit, protect, unprotect.
------------------------------------------------------------ */
-
-// OS page align within a given area, either conservative (pages inside the area only),
-// or not (straddling pages outside the area is possible)
-static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
-  mi_assert(addr != NULL && size > 0);
-  if (newsize != NULL) *newsize = 0;
-  if (size == 0 || addr == NULL) return NULL;
-
-  // page align conservatively within the range, or liberally straddling pages outside the range
-  void* start = (conservative ? _mi_align_up_ptr(addr, _mi_os_page_size())
-                              : _mi_align_down_ptr(addr, _mi_os_page_size()));
-  void* end   = (conservative ? _mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
-                              : _mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
-  ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
-  if (diff <= 0) return NULL;
-
-  mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size));
-  if (newsize != NULL) *newsize = (size_t)diff;
-  return start;
-}
-
-static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
-  return mi_os_page_align_areax(true, addr, size, newsize);
-}
-
-bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {
-  if (is_zero != NULL) { *is_zero = false; }
-  mi_os_stat_counter_increase(commit_calls, 1);
-
-  // page align range
-  size_t csize;
-  void* start = mi_os_page_align_areax(false /* conservative? */, addr, size, &csize);
-  if (csize == 0) return true;
-
-  // commit
-  bool os_is_zero = false;
-  int err = _mi_prim_commit(start, csize, &os_is_zero);
-  if (err != 0) {
-    _mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
-    return false;
-  }
-  if (os_is_zero && is_zero != NULL) {
-    *is_zero = true;
-    mi_assert_expensive(mi_mem_is_zero(start, csize));
-  }
-  // note: the following seems required for asan (otherwise `mimalloc-test-stress` fails)
-  #ifdef MI_TRACK_ASAN
-  if (os_is_zero) { mi_track_mem_defined(start,csize); }
-             else { mi_track_mem_undefined(start,csize); }
-  #endif
-  mi_os_stat_increase(committed, stat_size);  // use size for precise commit vs. decommit
-  return true;
-}
-
-bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
-  return _mi_os_commit_ex(addr, size, is_zero, size);
-}
-
-static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
-  mi_assert_internal(needs_recommit!=NULL);
-
-  // page align
-  size_t csize;
-  void* start = mi_os_page_align_area_conservative(addr, size, &csize);
-  if (csize == 0) return true;
-
-  // decommit
-  *needs_recommit = true;
-  int err = _mi_prim_decommit(start,csize,needs_recommit);
-  if (err != 0) {
-    _mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
-  }
-  else if (*needs_recommit) {
-    mi_os_stat_decrease(committed, stat_size);
-  }
-  mi_assert_internal(err == 0);
-  return (err == 0);
-}
-
-bool _mi_os_decommit(void* addr, size_t size) {
-  bool needs_recommit;
-  return mi_os_decommit_ex(addr, size, &needs_recommit, size);
-}
-
-
-// Signal to the OS that the address range is no longer in use
-// but may be used later again. This will release physical memory
-// pages and reduce swapping while keeping the memory committed.
-// We page align to a conservative area inside the range to reset.
-bool _mi_os_reset(void* addr, size_t size) {
-  // page align conservatively within the range
-  size_t csize;
-  void* start = mi_os_page_align_area_conservative(addr, size, &csize);
-  if (csize == 0) return true;  // || _mi_os_is_huge_reserved(addr)
-  mi_os_stat_counter_increase(reset, csize);
-  mi_os_stat_counter_increase(reset_calls, 1);
-
-  #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN
-  memset(start, 0, csize); // pretend it is eagerly reset
-  #endif
-
-  int err = _mi_prim_reset(start, csize);
-  if (err != 0) {
-    _mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
-  }
-  return (err == 0);
-}
-
-
-void _mi_os_reuse( void* addr, size_t size ) {
-  // page align conservatively within the range
-  size_t csize = 0;
-  void* const start = mi_os_page_align_area_conservative(addr, size, &csize);
-  if (csize == 0) return;
-  const int err = _mi_prim_reuse(start, csize);
-  if (err != 0) {
-    _mi_warning_message("cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
-  }
-}
-
-// either resets or decommits memory, returns true if the memory needs
-// to be recommitted if it is to be re-used later on.
-bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)
-{
-  if (mi_option_get(mi_option_purge_delay) < 0) return false;  // is purging allowed?
-  mi_os_stat_counter_increase(purge_calls, 1);
-  mi_os_stat_counter_increase(purged, size);
-
-  if (commit_fun != NULL) {
-    bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);
-    return decommitted; // needs_recommit?
-  }
-  else if (mi_option_is_enabled(mi_option_purge_decommits) &&   // should decommit?
-           !_mi_preloading())                                   // don't decommit during preloading (unsafe)
-  {
-    bool needs_recommit = true;
-    mi_os_decommit_ex(p, size, &needs_recommit, stat_size);
-    return needs_recommit;
-  }
-  else {
-    if (allow_reset) {  // this can sometimes be not allowed if the range is not fully committed (on Windows, we cannot reset uncommitted memory)
-      _mi_os_reset(p, size);
-    }
-    return false;  // needs no recommit
-  }
-}
-
-// either resets or decommits memory, returns true if the memory needs
-// to be recommitted if it is to be re-used later on.
-bool _mi_os_purge(void* p, size_t size) {
-  return _mi_os_purge_ex(p, size, true, size, NULL, NULL);
-}
-
-
-// Protect a region in memory to be not accessible.
-static  bool mi_os_protectx(void* addr, size_t size, bool protect) {
-  // page align conservatively within the range
-  size_t csize = 0;
-  void* start = mi_os_page_align_area_conservative(addr, size, &csize);
-  if (csize == 0) return false;
-  /*
-  if (_mi_os_is_huge_reserved(addr)) {
-	  _mi_warning_message("cannot mprotect memory allocated in huge OS pages\n");
-  }
-  */
-  int err = _mi_prim_protect(start,csize,protect);
-  if (err != 0) {
-    _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize);
-  }
-  return (err == 0);
-}
-
-bool _mi_os_protect(void* addr, size_t size) {
-  return mi_os_protectx(addr, size, true);
-}
-
-bool _mi_os_unprotect(void* addr, size_t size) {
-  return mi_os_protectx(addr, size, false);
-}
-
-
-
-/* ----------------------------------------------------------------------------
-Support for allocating huge OS pages (1Gib) that are reserved up-front
-and possibly associated with a specific NUMA node. (use `numa_node>=0`)
------------------------------------------------------------------------------*/
-#define MI_HUGE_OS_PAGE_SIZE  (MI_GiB)
-
-
-#if (MI_INTPTR_SIZE >= 8)
-// To ensure proper alignment, use our own area for huge OS pages
-static mi_decl_cache_align _Atomic(uintptr_t)  mi_huge_start; // = 0
-
-// Claim an aligned address range for huge pages
-static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
-  if (total_size != NULL) *total_size = 0;
-  const size_t size = pages * MI_HUGE_OS_PAGE_SIZE;
-
-  uintptr_t start = 0;
-  uintptr_t end = 0;
-  uintptr_t huge_start = mi_atomic_load_relaxed(&mi_huge_start);
-  do {
-    start = huge_start;
-    if (start == 0) {
-      // Initialize the start address after the 32TiB area
-      start = ((uintptr_t)8 << 40);   // 8TiB virtual start address
-      #if (MI_SECURE>0 || MI_DEBUG==0)  // security: randomize start of huge pages unless in debug mode
-      mi_theap_t* const theap = _mi_theap_default();     // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
-      if (mi_theap_is_initialized(theap)) {              // todo: or no hint at all if we lack randomness?
-        const uintptr_t r = _mi_theap_random_next(theap);
-        start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF));  // (randomly 12bits)*1GiB == between 0 to 4TiB
-      }
-      else {
-        _mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, start);
-      }
-      #endif
-    }
-    end = start + size;
-  } while (!mi_atomic_cas_weak_acq_rel(&mi_huge_start, &huge_start, end));
-
-  if (total_size != NULL) *total_size = size;
-  return (uint8_t*)start;
-}
-#else
-static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
-  MI_UNUSED(pages);
-  if (total_size != NULL) *total_size = 0;
-  return NULL;
-}
-#endif
-
-// Allocate MI_ARENA_SLICE_ALIGN aligned huge pages
-void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {
-  *memid = _mi_memid_none();
-  if (psize != NULL) *psize = 0;
-  if (pages_reserved != NULL) *pages_reserved = 0;
-  size_t size = 0;
-  uint8_t* const start = mi_os_claim_huge_pages(pages, &size);
-  if (start == NULL) return NULL; // or 32-bit systems
-
-  // Allocate one page at the time but try to place them contiguously
-  // We allocate one page at the time to be able to abort if it takes too long
-  // or to at least allocate as many as available on the system.
-  mi_msecs_t start_t = _mi_clock_start();
-  size_t page = 0;
-  bool all_zero = true;
-  while (page < pages) {
-    // allocate a page
-    bool is_zero = false;
-    void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE);
-    void* p = NULL;
-    int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p);
-    if (!is_zero) { all_zero = false;  }
-    if (err != 0) {
-      _mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE);
-      break;
-    }
-
-    // Did we succeed at a contiguous address?
-    if (p != addr) {
-      // no success, issue a warning and break
-      if (p != NULL) {
-        _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
-        mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, NULL);
-      }
-      break;
-    }
-
-    // success, record it
-    page++;  // increase before timeout check (see issue #711)
-    mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);
-    mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);
-
-    // check for timeout
-    if (max_msecs > 0) {
-      mi_msecs_t elapsed = _mi_clock_end(start_t);
-      if (page >= 1) {
-        mi_msecs_t estimate = ((elapsed / (page+1)) * pages);
-        if (estimate > 2*max_msecs) { // seems like we are going to timeout, break
-          elapsed = max_msecs + 1;
-        }
-      }
-      if (elapsed > max_msecs) {
-        _mi_warning_message("huge OS page allocation timed out (after allocating %zu page(s))\n", page);
-        break;
-      }
-    }
-  }
-  const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE;
-  mi_assert_internal(allocated <= size);
-  if (pages_reserved != NULL) { *pages_reserved = page; }
-  if (psize != NULL) { *psize = allocated; }
-  if (page != 0) {
-    mi_assert(start != NULL);
-    *memid = _mi_memid_create_os(start, allocated, true /* is committed */, all_zero, true /* is_large */);
-    memid->memkind = MI_MEM_OS_HUGE;
-    mi_assert(memid->is_pinned);
-    #ifdef MI_TRACK_ASAN
-    if (all_zero) { mi_track_mem_defined(start,allocated); }
-    #endif
-  }
-  return (page == 0 ? NULL : start);
-}
-
-// free every huge page in a range individually (as we allocated per page)
-// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.
-static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc) {
-  if (p==NULL || size==0) return;
-  uint8_t* base = (uint8_t*)p;
-  while (size >= MI_HUGE_OS_PAGE_SIZE) {
-    mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, subproc);
-    size -= MI_HUGE_OS_PAGE_SIZE;
-    base += MI_HUGE_OS_PAGE_SIZE;
-  }
-}
-
-
-/* ----------------------------------------------------------------------------
-Support NUMA aware allocation
------------------------------------------------------------------------------*/
-
-static _Atomic(size_t) mi_numa_node_count; // = 0   // cache the node count
-
-int _mi_os_numa_node_count(void) {
-  size_t count = mi_atomic_load_acquire(&mi_numa_node_count);
-  if mi_unlikely(count == 0) {
-    long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly?
-    if (ncount > 0 && ncount < INT_MAX) {
-      count = (size_t)ncount;
-    }
-    else {
-      const size_t n = _mi_prim_numa_node_count(); // or detect dynamically
-      if (n == 0 || n > INT_MAX) { count = 1; }
-                            else { count = n; }
-    }
-    mi_atomic_store_release(&mi_numa_node_count, count); // save it
-    if (count>1) { _mi_verbose_message("using %zd numa regions\n", count); }
-  }
-  mi_assert_internal(count > 0 && count <= INT_MAX);
-  return (int)count;
-}
-
-static int mi_os_numa_node_get(void) {
-  int numa_count = _mi_os_numa_node_count();
-  if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0
-  // never more than the node count and >= 0
-  const size_t n = _mi_prim_numa_node();
-  int numa_node = (n < INT_MAX ? (int)n : 0);
-  if (numa_node >= numa_count) { numa_node = numa_node % numa_count; }
-  return numa_node;
-}
-
-int _mi_os_numa_node(void) {
-  if mi_likely(mi_atomic_load_relaxed(&mi_numa_node_count) == 1) {
-    return 0;
-  }
-  else {
-    return mi_os_numa_node_get();
-  }
-}
-
-
-/* ----------------------------------------------------------------------------
-  Public API
------------------------------------------------------------------------------*/
-#if 0
-mi_decl_export void* mi_os_alloc(size_t size, bool commit, size_t* full_size) {
-  return mi_os_alloc_aligned(size, mi_os_mem_config.alloc_granularity, commit, NULL, full_size);
-}
-
-static void* mi_os_alloc_aligned_ex(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) {
-  mi_memid_t memid = _mi_memid_none();
-  void* p = _mi_os_alloc_aligned(size, alignment, commit, allow_large, &memid);
-  if (p == NULL) return p;
-  if (is_committed != NULL) { *is_committed = memid.initially_committed;  }
-  if (is_pinned != NULL) { *is_pinned = memid.is_pinned;  }
-  if (base != NULL) { *base = memid.mem.os.base;  }
-  if (full_size != NULL) { *full_size = memid.mem.os.size;  }
-  if (!memid.initially_zero && memid.initially_committed) {
-    _mi_memzero_aligned(memid.mem.os.base, memid.mem.os.size);
-  }
-  return p;
-}
-
-mi_decl_export void* mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, void** base, size_t* full_size) {
-  return mi_os_alloc_aligned_ex(size, alignment, commit, false, NULL, NULL, base, full_size);
-}
-
-mi_decl_export void* mi_os_alloc_aligned_allow_large(size_t size, size_t alignment, bool commit, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) {
-  return mi_os_alloc_aligned_ex(size, alignment, commit, true, is_committed, is_pinned, base, full_size);
-}
-
-mi_decl_export void  mi_os_free(void* p, size_t size) {
-  if (p==NULL || size == 0) return;
-  mi_memid_t memid = _mi_memid_create_os(p, size, true, false, false);
-  _mi_os_free(p, size, memid);
-}
-
-mi_decl_export void  mi_os_commit(void* p, size_t size) {
-  _mi_os_commit(p, size, NULL);
-}
-
-mi_decl_export void  mi_os_decommit(void* p, size_t size) {
-  _mi_os_decommit(p, size);
-}
-#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-map.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-map.c
deleted file mode 100644
index acc0ee9f..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-map.c
+++ /dev/null
@@ -1,447 +0,0 @@
-/*----------------------------------------------------------------------------
-Copyright (c) 2023-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "bitmap.h"
-
-static void mi_page_map_cannot_commit(void) {
-  _mi_warning_message("unable to commit the allocation page-map on-demand\n" );
-}
-
-#if MI_PAGE_MAP_FLAT
-
-// The page-map contains a byte for each 64kb slice in the address space.
-// For an address `a` where `ofs = _mi_page_map[a >> 16]`:
-// 0 = unused
-// 1 = the slice at `a & ~0xFFFF` is a mimalloc page.
-// 1 < ofs <= 127 = the slice is part of a page, starting at `(((a>>16) - ofs - 1) << 16)`.
-//
-// 1 byte per slice => 1 TiB address space needs a 2^14 * 2^16 = 16 MiB page map.
-// A full 256 TiB address space (48 bit) needs a 4 GiB page map.
-// A full 4 GiB address space (32 bit) needs only a 64 KiB page map.
-
-mi_decl_cache_align uint8_t* _mi_page_map = NULL;
-static void*        mi_page_map_max_address = NULL;
-static mi_memid_t   mi_page_map_memid;
-
-#define MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT   MI_ARENA_SLICE_SIZE
-static mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entries
-
-mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count);
-
-bool _mi_page_map_init(void) {
-  size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);
-  if (vbits == 0) {
-    vbits = _mi_os_virtual_address_bits();
-    #if MI_ARCH_X64  // canonical address is limited to the first 128 TiB
-    if (vbits >= 48) { vbits = 47; }
-    #endif
-  }
-  if (vbits < MI_ARENA_SLICE_SHIFT) {
-    vbits = MI_ARENA_SLICE_SHIFT;
-  }
-
-  // Allocate the page map and commit bits
-  mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
-  const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT));
-  const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems?
-  const size_t commit_bits = _mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);
-  const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL));
-  const size_t reserve_size = bitmap_size + page_map_size;
-  uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);
-  if (base==NULL) {
-    _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);
-    return false;
-  }
-  if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {
-    _mi_warning_message("internal: the page map was committed but not zero initialized!\n");
-    _mi_memzero_aligned(base, reserve_size);
-  }
-  if (bitmap_size > 0) {
-    mi_page_map_commit = (mi_bitmap_t*)base;
-    if (!_mi_os_commit(mi_page_map_commit, bitmap_size, NULL)) {
-      mi_page_map_cannot_commit();
-      return false;
-    }
-    mi_bitmap_init(mi_page_map_commit, commit_bits, true);
-  }
-  _mi_page_map = base + bitmap_size;
-
-  // commit the first part so NULL pointers get resolved without an access violation
-  if (!commit) {
-    if (!mi_page_map_ensure_committed(0, 1)) {
-      mi_page_map_cannot_commit();
-      return false;
-    }
-  }
-  _mi_page_map[0] = 1; // so _mi_ptr_page(NULL) == NULL
-  mi_assert_internal(_mi_ptr_page(NULL)==NULL);
-  return true;
-}
-
-void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {
-  mi_assert_internal(subproc != NULL);
-  mi_assert_internal(_mi_page_map != NULL);
-  if (_mi_page_map == NULL) return;
-  _mi_os_free_ex(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);
-  _mi_page_map = NULL;
-  mi_page_map_commit = NULL;
-  mi_page_map_max_address = NULL;
-  mi_page_map_memid = _mi_memid_none();
-}
-
-
-static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {
-  // is the page map area that contains the page address committed?
-  // we always set the commit bits so we can track what ranges are in-use.
-  // we only actually commit if the map wasn't committed fully already.
-  if (mi_page_map_commit != NULL) {
-    const size_t commit_idx = idx / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
-    const size_t commit_idx_hi = (idx + slice_count - 1) / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
-    for (size_t i = commit_idx; i <= commit_idx_hi; i++) {  // per bit to avoid crossing over bitmap chunks
-      if (mi_bitmap_is_clear(mi_page_map_commit, i)) {
-        // this may race, in which case we do multiple commits (which is ok)
-        bool is_zero;
-        uint8_t* const start = _mi_page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);
-        const size_t   size  = MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
-        if (!_mi_os_commit(start, size, &is_zero)) {
-          mi_page_map_cannot_commit();
-          return false;
-        }
-        if (!is_zero && !mi_page_map_memid.initially_zero) { _mi_memzero(start, size); }
-        mi_bitmap_set(mi_page_map_commit, i);
-      }
-    }
-  }
-  #if MI_DEBUG > 0
-  _mi_page_map[idx] = 0;
-  _mi_page_map[idx+slice_count-1] = 0;
-  #endif
-  return true;
-}
-
-
-static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* slice_count) {
-  size_t page_size;
-  *page_start = mi_page_area(page, &page_size);
-  if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; }  // furthest interior pointer
-  *slice_count = mi_slice_count_of_size(page_size) + ((*page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks
-  return _mi_page_map_index(page);
-}
-
-bool _mi_page_map_register(mi_page_t* page) {
-  mi_assert_internal(page != NULL);
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_page_map != NULL);  // should be initialized before multi-thread access!
-  if mi_unlikely(_mi_page_map == NULL) {
-    if (!_mi_page_map_init()) return false;
-  }
-  mi_assert(_mi_page_map!=NULL);
-  uint8_t* page_start;
-  size_t   slice_count;
-  const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);
-
-  if (!mi_page_map_ensure_committed(idx, slice_count)) {
-    return false;
-  }
-
-  // set the offsets
-  for (size_t i = 0; i < slice_count; i++) {
-    mi_assert_internal(i < 128);
-    _mi_page_map[idx + i] = (uint8_t)(i+1);
-  }
-  return true;
-}
-
-void _mi_page_map_unregister(mi_page_t* page) {
-  mi_assert_internal(_mi_page_map != NULL);
-  // get index and count
-  uint8_t* page_start;
-  size_t   slice_count;
-  const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);
-  // unset the offsets
-  _mi_memzero(_mi_page_map + idx, slice_count);
-}
-
-void _mi_page_map_unregister_range(void* start, size_t size) {
-  const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
-  const uintptr_t index = _mi_page_map_index(start);
-  // todo: scan the commit bits and clear only those ranges?
-  if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total;
-    return;
-  }
-  _mi_memzero(&_mi_page_map[index], slice_count);
-}
-
-
-mi_page_t* _mi_safe_ptr_page(const void* p) {
-  if mi_unlikely(p >= mi_page_map_max_address) return NULL;
-  const uintptr_t idx = _mi_page_map_index(p);
-  if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL;
-  const uintptr_t ofs = _mi_page_map[idx];
-  if mi_unlikely(ofs == 0) return NULL;
-  return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT);
-}
-
-mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
-  return (_mi_safe_ptr_page(p) != NULL);
-}
-
-#else
-
-// A 2-level page map
-#define MI_PAGE_MAP_SUB_SIZE          (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*))
-#define MI_PAGE_MAP_ENTRIES_PER_CBIT  (MI_PAGE_MAP_COUNT < MI_BFIELD_BITS ? 1 : (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS))
-
-mi_decl_cache_align _Atomic(mi_submap_t)* _mi_page_map;
-static size_t       mi_page_map_count;
-static void*        mi_page_map_max_address;
-static mi_memid_t   mi_page_map_memid;
-static mi_lock_t    mi_page_map_lock;
-
-// divide the main map in 64 (`MI_BFIELD_BITS`) parts commit those parts on demand
-static _Atomic(mi_bfield_t)  mi_page_map_commit;
-
-mi_decl_nodiscard static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) {
-  mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit);
-  const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT;
-  mi_assert_internal(bit_idx < MI_BFIELD_BITS);
-  if (pbit_idx != NULL) { *pbit_idx = bit_idx; }
-  return ((commit & (MI_ZU(1) << bit_idx)) != 0);
-}
-
-mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, mi_submap_t* submap) {
-  mi_assert_internal(submap!=NULL && *submap==NULL);
-  size_t bit_idx;
-  if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) {
-    uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT];
-    if (!_mi_os_commit(start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_submap_t), NULL)) {
-      mi_page_map_cannot_commit();
-      return false;
-    }
-    mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx);
-  }
-  *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // acquire _mi_page_map_at(idx);
-  return true;
-}
-
-// initialize the page map
-bool _mi_page_map_init(void) {
-  size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);
-  if (vbits == 0) {
-    vbits = _mi_os_virtual_address_bits();
-    #if MI_ARCH_X64  // canonical address is limited to the first 128 TiB
-    if (vbits >= 48) { vbits = 47; }
-    #endif
-  }
-  if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) {
-    vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT;
-  }
-
-  // Allocate the page map and commit bits
-  mi_assert(MI_MAX_VABITS >= vbits);
-  mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
-  mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
-  mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT);
-  const size_t os_page_size = _mi_os_page_size();
-  const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size);
-  const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
-  const size_t reserve_size = page_map_size + submap_size;
-  #if MI_SECURE
-  const bool commit = true;  // the whole page map is valid and we can reliably check any pointer
-  #else
-  const bool commit = page_map_size <= 64*MI_KiB ||
-                      mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit();
-  #endif
-  _mi_page_map = (_Atomic(mi_page_t**)*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);
-  if (_mi_page_map==NULL) {
-    _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);
-    return false;
-  }
-  if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {
-    _mi_warning_message("internal: the page map was committed but not zero initialized!\n");
-    _mi_memzero_aligned(_mi_page_map, page_map_size);
-  }
-  mi_atomic_store_release(&mi_page_map_commit, (mi_page_map_memid.initially_committed ? ~MI_ZU(0) : MI_ZU(0)));
-
-  // ensure there is a submap for the NULL address
-  mi_page_t** const sub0 = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size);  // we reserved a submap part at the end already
-  if (!mi_page_map_memid.initially_committed) {
-    if (!_mi_os_commit(sub0, submap_size, NULL)) {  // commit full submap (issue #1087)
-      mi_page_map_cannot_commit();
-      return false;
-    }
-  }
-  if (!mi_page_map_memid.initially_zero) {     // initialize low addresses with NULL
-    _mi_memzero_aligned(sub0, submap_size);
-  }
-  mi_submap_t nullsub = NULL;
-  if (!mi_page_map_ensure_committed(0,&nullsub)) {
-    mi_page_map_cannot_commit();
-    return false;
-  }
-  mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[0], sub0);
-  mi_lock_init(&mi_page_map_lock);             // initialize late in case the lock init causes allocation
-  
-  mi_assert_internal(_mi_ptr_page(NULL)==NULL);
-  return true;
-}
-
-
-void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {
-  mi_assert_internal(subproc != NULL);
-  mi_assert_internal(_mi_page_map != NULL);
-  if (_mi_page_map == NULL) return;
-  mi_lock_done(&mi_page_map_lock);  
-  for (size_t idx = 1; idx < mi_page_map_count; idx++) {  // skip entry 0 (as we allocate that submap at the end of the page_map)
-    // free all sub-maps
-    if (mi_page_map_is_committed(idx, NULL)) {
-      mi_submap_t sub = _mi_page_map_at(idx);
-      if (sub != NULL) {
-        mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false);
-        _mi_os_free_ex(memid.mem.os.base, memid.mem.os.size, true, memid, subproc);
-        mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[idx], NULL);
-      }
-    }
-  }
-  _mi_os_free_ex(_mi_page_map, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);
-  _mi_page_map = NULL;
-  mi_page_map_count = 0;
-  mi_page_map_memid = _mi_memid_none();
-  mi_page_map_max_address = NULL;
-  mi_atomic_store_release(&mi_page_map_commit, (mi_bfield_t)0);
-}
-
-
-mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap_t* submap) {
-  mi_assert_internal(submap!=NULL && *submap==NULL);
-  mi_submap_t sub = NULL;
-  if (!mi_page_map_ensure_committed(idx, &sub)) {
-    return false;
-  }
-  if mi_unlikely(sub == NULL) {
-    // sub map not yet allocated, alloc now
-    mi_lock(&mi_page_map_lock) 
-    {
-      sub = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // reload
-      if (sub==NULL) // not yet allocated by another thread?      
-      {
-        mi_memid_t memid;
-        const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
-        sub = (mi_submap_t)_mi_os_zalloc(submap_size, &memid);        
-        if (sub==NULL) {
-          _mi_warning_message("internal error: unable to extend the page map\n");          
-        }
-        else {
-          mi_submap_t expect = NULL;
-          if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &_mi_page_map[idx], &expect, sub)) {
-            // another thread already allocated it.. free and continue
-            _mi_os_free(sub, submap_size, memid);
-            sub = expect;
-          }
-        }
-      }
-    }
-    if (sub==NULL) return false; // unable to allocate the submap..
-  }
-  mi_assert_internal(sub!=NULL);
-  *submap = sub;
-  return true;
-}
-
-static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
-  // is the page map area that contains the page address committed?
-  while (slice_count > 0) {
-    mi_submap_t sub = NULL;
-    if (!mi_page_map_ensure_submap_at(idx, &sub)) {
-      return false;
-    };
-    mi_assert_internal(sub!=NULL);
-    // set the offsets for the page
-    while (slice_count > 0 && sub_idx < MI_PAGE_MAP_SUB_COUNT) {
-      sub[sub_idx] = page;
-      slice_count--;
-      sub_idx++;
-    }
-    idx++; // potentially wrap around to the next idx
-    sub_idx = 0;
-  }
-  return true;
-}
-
-static bool mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
-  if mi_unlikely(!mi_page_map_set_range_prim(page,idx,sub_idx,slice_count)) {
-    // failed to commit, call again to reset the page pointer if needed
-    if (page!=NULL) {
-      mi_page_map_set_range_prim(NULL,idx,sub_idx,slice_count);
-    }
-    return false;
-  }
-  return true;
-}
-
-static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slice_count) {
-  size_t page_size;
-  uint8_t* page_start = mi_page_area(page, &page_size);
-  if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; }  // furthest interior pointer
-  *slice_count = mi_slice_count_of_size(page_size) + ((page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks
-  return _mi_page_map_index(page_start, sub_idx);
-}
-
-bool _mi_page_map_register(mi_page_t* page) {
-  mi_assert_internal(page != NULL);
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_page_map != NULL);  // should be initialized before multi-thread access!
-  if mi_unlikely(_mi_page_map == NULL) {
-    if (!_mi_page_map_init()) return false;
-  }
-  mi_assert(_mi_page_map!=NULL);
-  size_t   slice_count;
-  size_t   sub_idx;
-  const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
-  return mi_page_map_set_range(page, idx, sub_idx, slice_count);
-}
-
-void _mi_page_map_unregister(mi_page_t* page) {
-  mi_assert_internal(_mi_page_map != NULL);
-  mi_assert_internal(page != NULL);
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  if mi_unlikely(_mi_page_map == NULL) return;
-  // get index and count
-  size_t slice_count;
-  size_t sub_idx;
-  const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
-  // unset the offsets
-  mi_page_map_set_range(NULL, idx, sub_idx, slice_count);
-}
-
-void _mi_page_map_unregister_range(void* start, size_t size) {
-  if mi_unlikely(_mi_page_map == NULL) return;
-  const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
-  size_t sub_idx;
-  const uintptr_t idx = _mi_page_map_index(start, &sub_idx);
-  mi_page_map_set_range(NULL, idx, sub_idx, slice_count);  // todo: avoid committing if not already committed?
-}
-
-// Return NULL for invalid pointers
-mi_page_t* _mi_safe_ptr_page(const void* p) {
-  if (p==NULL) return NULL;
-  if mi_unlikely(p >= mi_page_map_max_address) return NULL;
-  size_t sub_idx;
-  const size_t idx = _mi_page_map_index(p,&sub_idx);
-  if mi_unlikely(!mi_page_map_is_committed(idx,NULL)) return NULL;
-  mi_page_t** const sub = _mi_page_map[idx];
-  if mi_unlikely(sub==NULL) return NULL;
-  return sub[sub_idx];
-}
-
-mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
-  return (_mi_safe_ptr_page(p) != NULL);
-}
-
-#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-queue.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-queue.c
deleted file mode 100644
index 731672b0..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page-queue.c
+++ /dev/null
@@ -1,458 +0,0 @@
-/*----------------------------------------------------------------------------
-Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* -----------------------------------------------------------
-  Definition of page queues for each block size
------------------------------------------------------------ */
-
-#ifndef MI_IN_PAGE_C
-#error "this file should be included from 'page.c'"
-// include to help an IDE
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/atomic.h"
-#endif
-
-/* -----------------------------------------------------------
-  Minimal alignment in machine words (i.e. `sizeof(void*)`)
------------------------------------------------------------ */
-
-#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE)
-  #error "define alignment for more than 4x word size for this platform"
-#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE)
-  #define MI_ALIGN4W   // 4 machine words minimal alignment
-#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE)
-  #define MI_ALIGN2W   // 2 machine words minimal alignment
-#else
-  // ok, default alignment is 1 word
-#endif
-
-
-/* -----------------------------------------------------------
-  Queue query
------------------------------------------------------------ */
-
-
-static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) {
-  return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+sizeof(uintptr_t)));
-}
-
-static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) {
-  return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+(2*sizeof(uintptr_t))));
-}
-
-static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) {
-  return (pq->block_size > MI_LARGE_MAX_OBJ_SIZE);
-}
-
-static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) {
-  return pq->count;
-}
-
-/* -----------------------------------------------------------
-  Bins
------------------------------------------------------------ */
-
-// Return the bin for a given field size.
-// Returns MI_BIN_HUGE if the size is too large.
-// We use `wsize` for the size in "machine word sizes",
-// i.e. byte size == `wsize*sizeof(void*)`.
-static mi_decl_noinline size_t mi_bin(size_t size) {
-  size_t wsize = _mi_wsize_from_size(size);
-#if defined(MI_ALIGN4W)
-  if mi_likely(wsize <= 4) {
-    return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
-  }
-#elif defined(MI_ALIGN2W)
-  if mi_likely(wsize <= 8) {
-    return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
-  }
-#else
-  if mi_likely(wsize <= 8) {
-    return (wsize == 0 ? 1 : wsize);
-  }
-#endif
-  else if mi_unlikely(wsize > MI_LARGE_MAX_OBJ_WSIZE) {
-    return MI_BIN_HUGE;
-  }
-  else {
-    #if defined(MI_ALIGN4W)
-    if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes
-    #endif
-    wsize--;
-    // find the highest bit
-    const size_t b = (MI_SIZE_BITS - 1 - mi_clz(wsize));  // note: wsize != 0
-    // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).
-    // - adjust with 3 because we use do not round the first 8 sizes
-    //   which each get an exact bin
-    const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3;
-    mi_assert_internal(bin > 0 && bin < MI_BIN_HUGE);
-    return bin;
-  }
-}
-
-
-
-/* -----------------------------------------------------------
-  Queue of pages with free blocks
------------------------------------------------------------ */
-
-size_t _mi_bin(size_t size) {
-  return mi_bin(size);
-}
-
-size_t _mi_bin_size(size_t bin) {
-  mi_assert_internal(bin <= MI_BIN_HUGE);
-  return _mi_theap_empty.pages[bin].block_size;
-}
-
-// Good size for allocation
-mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept {
-  if (size <= MI_LARGE_MAX_OBJ_SIZE) {
-    return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE));
-  }
-  else if (size <= MI_MAX_ALLOC_SIZE) {
-    return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size());
-  }
-  else {
-    return size;
-  }
-}
-
-#if (MI_DEBUG>1)
-static bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) {
-  mi_assert_internal(page != NULL);
-  mi_page_t* list = queue->first;
-  while (list != NULL) {
-    mi_assert_internal(list->next == NULL || list->next->prev == list);
-    mi_assert_internal(list->prev == NULL || list->prev->next == list);
-    if (list == page) break;
-    list = list->next;
-  }
-  return (list == page);
-}
-
-#endif
-
-#if (MI_DEBUG>1)
-static bool mi_theap_contains_queue(const mi_theap_t* theap, const mi_page_queue_t* pq) {
-  return (pq >= &theap->pages[0] && pq <= &theap->pages[MI_BIN_FULL]);
-}
-#endif
-
-bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) {
-  MI_UNUSED_RELEASE(theap);
-  if (pq==NULL) return false;
-  size_t count = 0; MI_UNUSED_RELEASE(count);
-  mi_page_t* prev = NULL; MI_UNUSED_RELEASE(prev);
-  for (mi_page_t* page = pq->first; page != NULL; page = page->next) {
-    mi_assert_internal(page->prev == prev);
-    if (mi_page_is_in_full(page)) {
-      mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2);
-    }
-    else if (mi_page_is_huge(page)) {
-      mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1);
-    }
-    else {
-      mi_assert_internal(mi_page_block_size(page) == pq->block_size);
-    }
-    mi_assert_internal(page->theap == theap);
-    if (page->next == NULL) {
-      mi_assert_internal(pq->last == page);
-    }
-    count++;
-    prev = page;
-  }
-  mi_assert_internal(pq->count == count);
-  return true;
-}
-
-static size_t mi_page_bin(const mi_page_t* page) {
-  const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))));
-  mi_assert_internal(bin <= MI_BIN_FULL);
-  return bin;
-}
-
-// returns the page bin without using MI_BIN_FULL for statistics
-size_t _mi_page_stats_bin(const mi_page_t* page) {
-  const size_t bin = (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)));
-  mi_assert_internal(bin <= MI_BIN_HUGE);
-  return bin;
-}
-
-static mi_page_queue_t* mi_theap_page_queue_of(mi_theap_t* theap, const mi_page_t* page) {
-  mi_assert_internal(theap!=NULL);
-  const size_t bin = mi_page_bin(page);
-  mi_page_queue_t* pq = &theap->pages[bin];
-  mi_assert_internal((mi_page_block_size(page) == pq->block_size) ||
-                       (mi_page_is_huge(page) && mi_page_queue_is_huge(pq)) ||
-                         (mi_page_is_in_full(page) && mi_page_queue_is_full(pq)));
-  return pq;
-}
-
-static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) {
-  mi_theap_t* theap = mi_page_theap(page);
-  mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
-  mi_assert_expensive(mi_page_queue_contains(pq, page));
-  return pq;
-}
-
-// The current small page array is for efficiency and for each
-// small size (up to 256) it points directly to the page for that
-// size without having to compute the bin. This means when the
-// current free page queue is updated for a small bin, we need to update a
-// range of entries in `_mi_page_small_free`.
-static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_queue_t* pq) {
-  mi_assert_internal(mi_theap_contains_queue(theap,pq));
-  const size_t size = pq->block_size;
-  if (size > MI_SMALL_SIZE_MAX) return;
-
-  mi_page_t* page = pq->first;
-  if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty;
-
-  // find index in the right direct page array
-  const size_t idx = _mi_wsize_from_size(size);
-  mi_page_t** const pages_free = theap->pages_free_direct;
-  if (pages_free[idx] == page) return;  // already set
-
-  // find start slot
-  size_t start;
-  if (idx<=1) {
-    start = 0;
-  }
-  else {
-    // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped
-    mi_assert_internal(pq > &theap->pages[0]); // since idx > 1    
-    size_t bin = mi_bin(size);
-    const mi_page_queue_t* prev = pq - 1;
-    while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) {
-      prev--;
-    }
-    start = 1 + _mi_wsize_from_size(prev->block_size);
-    if (start > idx) start = idx;
-  }
-
-  // set size range to the right page
-  mi_assert(start <= idx);
-  for (size_t sz = start; sz <= idx; sz++) {
-    pages_free[sz] = page;
-  }
-}
-
-/*
-static bool mi_page_queue_is_empty(mi_page_queue_t* queue) {
-  return (queue->first == NULL);
-}
-*/
-
-static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) {
-  mi_assert_internal(page != NULL);
-  mi_assert_expensive(mi_page_queue_contains(queue, page));
-  mi_assert_internal(queue->count >= 1);
-  mi_assert_internal(mi_page_block_size(page) == queue->block_size ||
-                      (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||
-                        (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
-  mi_theap_t* theap = mi_page_theap(page);
-  if (page->prev != NULL) page->prev->next = page->next;
-  if (page->next != NULL) page->next->prev = page->prev;
-  if (page == queue->last)  queue->last = page->prev;
-  if (page == queue->first) {
-    queue->first = page->next;
-    // update first
-    mi_assert_internal(mi_theap_contains_queue(theap, queue));
-    mi_theap_queue_first_update(theap,queue);
-  }
-  theap->page_count--;
-  queue->count--;
-  page->next = NULL;
-  page->prev = NULL;
-  mi_page_set_in_full(page,false);
-}
-
-
-static void mi_page_queue_push(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
-  mi_assert_internal(mi_page_theap(page) == theap);
-  mi_assert_internal(!mi_page_queue_contains(queue, page));
-  #if MI_HUGE_PAGE_ABANDON
-  mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE);
-  #endif
-  mi_assert_internal(mi_page_block_size(page) == queue->block_size ||
-                      (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||
-                        (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
-
-  mi_page_set_in_full(page, mi_page_queue_is_full(queue));
-
-  page->next = queue->first;
-  page->prev = NULL;
-  if (queue->first != NULL) {
-    mi_assert_internal(queue->first->prev == NULL);
-    queue->first->prev = page;
-    queue->first = page;
-  }
-  else {
-    queue->first = queue->last = page;
-  }
-  queue->count++;
-
-  // update direct
-  mi_theap_queue_first_update(theap, queue);
-  theap->page_count++;
-}
-
-static void mi_page_queue_push_at_end(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
-  mi_assert_internal(mi_page_theap(page) == theap);
-  mi_assert_internal(!mi_page_queue_contains(queue, page));
-
-  mi_assert_internal(mi_page_block_size(page) == queue->block_size ||
-                      (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||
-                       (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
-
-  mi_page_set_in_full(page, mi_page_queue_is_full(queue));
-
-  page->prev = queue->last;
-  page->next = NULL;
-  if (queue->last != NULL) {
-    mi_assert_internal(queue->last->next == NULL);
-    queue->last->next = page;
-    queue->last = page;
-  }
-  else {
-    queue->first = queue->last = page;
-  }
-  queue->count++;
-
-  // update direct
-  if (queue->first == page) {
-    mi_theap_queue_first_update(theap, queue);
-  }
-  theap->page_count++;
-}
-
-static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
-  mi_assert_internal(mi_page_theap(page) == theap);
-  mi_assert_internal(mi_page_queue_contains(queue, page));
-  if (queue->first == page) return;
-  mi_page_queue_remove(queue, page);
-  mi_page_queue_push(theap, queue, page);
-  mi_assert_internal(queue->first == page);
-}
-
-static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) {
-  mi_assert_internal(page != NULL);
-  mi_assert_internal(from->count >= 1);
-  mi_assert_expensive(mi_page_queue_contains(from, page));
-  mi_assert_expensive(!mi_page_queue_contains(to, page));
-  const size_t bsize = mi_page_block_size(page);
-  MI_UNUSED(bsize);
-  mi_assert_internal((bsize == to->block_size && bsize == from->block_size) ||
-                     (bsize == to->block_size && mi_page_queue_is_full(from)) ||
-                     (bsize == from->block_size && mi_page_queue_is_full(to)) ||
-                     (mi_page_is_huge(page) && mi_page_queue_is_huge(to)) ||
-                     (mi_page_is_huge(page) && mi_page_queue_is_full(to)));
-
-  mi_theap_t* theap = mi_page_theap(page);
-
-  // delete from `from`
-  if (page->prev != NULL) page->prev->next = page->next;
-  if (page->next != NULL) page->next->prev = page->prev;
-  if (page == from->last)  from->last = page->prev;
-  if (page == from->first) {
-    from->first = page->next;
-    // update first
-    mi_assert_internal(mi_theap_contains_queue(theap, from));
-    mi_theap_queue_first_update(theap, from);
-  }
-  from->count--;
-
-  // insert into `to`
-  to->count++;
-  if (enqueue_at_end) {
-    // enqueue at the end
-    page->prev = to->last;
-    page->next = NULL;
-    if (to->last != NULL) {
-      mi_assert_internal(theap == mi_page_theap(to->last));
-      to->last->next = page;
-      to->last = page;
-    }
-    else {
-      to->first = page;
-      to->last = page;
-      mi_theap_queue_first_update(theap, to);
-    }
-  }
-  else {
-    if (to->first != NULL) {
-      // enqueue at 2nd place
-      mi_assert_internal(theap == mi_page_theap(to->first));
-      mi_page_t* next = to->first->next;
-      page->prev = to->first;
-      page->next = next;
-      to->first->next = page;
-      if (next != NULL) {
-        next->prev = page;
-      }
-      else {
-        to->last = page;
-      }
-    }
-    else {
-      // enqueue at the head (singleton list)
-      page->prev = NULL;
-      page->next = NULL;
-      to->first = page;
-      to->last = page;
-      mi_theap_queue_first_update(theap, to);
-    }
-  }
-
-  mi_page_set_in_full(page, mi_page_queue_is_full(to));
-}
-
-static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {
-  mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end */, page);
-}
-
-static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {
-  // note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`)
-  mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page);
-}
-
-// Only called from `mi_theap_absorb`.
-size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append) {
-  mi_assert_internal(mi_theap_contains_queue(theap,pq));
-  mi_assert_internal(pq->block_size == append->block_size);
-
-  if (append->first==NULL) return 0;
-
-  // set append pages to new theap and count
-  size_t count = 0;
-  for (mi_page_t* page = append->first; page != NULL; page = page->next) {
-    mi_page_set_theap(page, theap);
-    count++;
-  }
-  mi_assert_internal(count == append->count);
-
-  if (pq->last==NULL) {
-    // take over afresh
-    mi_assert_internal(pq->first==NULL);
-    pq->first = append->first;
-    pq->last = append->last;
-    mi_theap_queue_first_update(theap, pq);
-  }
-  else {
-    // append to end
-    mi_assert_internal(pq->last!=NULL);
-    mi_assert_internal(append->first!=NULL);
-    pq->last->next = append->first;
-    append->first->prev = pq->last;
-    pq->last = append->last;
-  }
-  pq->count += append->count;
-
-  return count;
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page.c
deleted file mode 100644
index 97d2fc27..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/page.c
+++ /dev/null
@@ -1,1039 +0,0 @@
-/*----------------------------------------------------------------------------
-Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* -----------------------------------------------------------
-  The core of the allocator. Every segment contains
-  pages of a certain block size. The main function
-  exported is `mi_malloc_generic`.
------------------------------------------------------------ */
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/atomic.h"
-#include "mimalloc/prim.h"
-
-/* -----------------------------------------------------------
-  Definition of page queues for each block size
------------------------------------------------------------ */
-
-#define MI_IN_PAGE_C
-#include "page-queue.c"
-#undef MI_IN_PAGE_C
-
-
-/* -----------------------------------------------------------
-  Page helpers
------------------------------------------------------------ */
-
-// Index a block in a page
-static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) {
-  MI_UNUSED(page);
-  mi_assert_internal(page != NULL);
-  mi_assert_internal(i <= page->reserved);
-  return (mi_block_t*)((uint8_t*)page_start + (i * block_size));
-}
-
-static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page);
-
-#if (MI_DEBUG>=3)
-static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) {
-  mi_assert_internal(_mi_ptr_page(page->page_start) == page);
-  const uint8_t* slice_start = mi_page_slice_start(page);
-  mi_assert_internal(_mi_is_aligned(slice_start,MI_PAGE_ALIGN));
-  size_t count = 0;
-  while (head != NULL) {
-    mi_assert_internal((uint8_t*)head - slice_start > (ptrdiff_t)MI_LARGE_PAGE_SIZE || page == _mi_ptr_page(head));
-    count++;
-    head = mi_block_next(page, head);
-  }
-  return count;
-}
-
-/*
-// Start of the page available memory
-static inline uint8_t* mi_page_area(const mi_page_t* page) {
-  return _mi_page_start(_mi_page_segment(page), page, NULL);
-}
-*/
-
-static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) {
-  size_t psize;
-  uint8_t* page_area = mi_page_area(page, &psize);
-  mi_block_t* start = (mi_block_t*)page_area;
-  mi_block_t* end   = (mi_block_t*)(page_area + psize);
-  while(p != NULL) {
-    if (p < start || p >= end) return false;
-    p = mi_block_next(page, p);
-  }
-#if MI_DEBUG>3 // generally too expensive to check this
-  if (page->free_is_zero) {
-    const size_t ubsize = mi_page_usable_block_size(page);
-    for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {
-      mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));
-    }
-  }
-#endif
-  return true;
-}
-
-static bool mi_page_is_valid_init(mi_page_t* page) {
-  mi_assert_internal(mi_page_block_size(page) > 0);
-  mi_assert_internal(page->used <= page->capacity);
-  mi_assert_internal(page->capacity <= page->reserved);
-
-  // const size_t bsize = mi_page_block_size(page);
-  // uint8_t* start = mi_page_start(page);
-  //mi_assert_internal(start + page->capacity*page->block_size == page->top);
-
-  mi_assert_internal(mi_page_list_is_valid(page,page->free));
-  mi_assert_internal(mi_page_list_is_valid(page,page->local_free));
-
-  #if MI_DEBUG>3 // generally too expensive to check this
-  if (page->free_is_zero) {
-    const size_t ubsize = mi_page_usable_block_size(page);
-    for(mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) {
-      mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));
-    }
-  }
-  #endif
-
-  #if !MI_TRACK_ENABLED && !MI_TSAN
-  mi_block_t* tfree = mi_page_thread_free(page);
-  mi_assert_internal(mi_page_list_is_valid(page, tfree));
-  //size_t tfree_count = mi_page_list_count(page, tfree);
-  //mi_assert_internal(tfree_count <= page->thread_freed + 1);
-  #endif
-
-  size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free);
-  mi_assert_internal(page->used + free_count == page->capacity);
-
-  return true;
-}
-
-extern mi_decl_hidden bool _mi_process_is_initialized;             // has mi_process_init been called?
-
-bool _mi_page_is_valid(mi_page_t* page) {
-  mi_assert_internal(mi_page_is_valid_init(page));
-  #if MI_SECURE
-  mi_assert_internal(page->keys[0] != 0);
-  #endif
-  if (!mi_page_is_abandoned(page)) {
-    //mi_assert_internal(!_mi_process_is_initialized);
-    {
-      mi_page_queue_t* pq = mi_page_queue_of(page);
-      mi_assert_internal(mi_page_queue_contains(pq, page));
-      mi_assert_internal(pq->block_size==mi_page_block_size(page) || mi_page_is_huge(page) || mi_page_is_in_full(page));
-      // mi_assert_internal(mi_theap_contains_queue(mi_page_theap(page),pq));
-    }
-  }
-  return true;
-}
-#endif
-
-
-/* -----------------------------------------------------------
-  Page collect the `local_free` and `thread_free` lists
------------------------------------------------------------ */
-
-static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head)
-{
-  if (head == NULL) return;
-
-  // find the last block in the list -- also to get a proper use count (without data races)
-  size_t max_count = page->capacity; // cannot collect more than capacity
-  size_t count = 1;
-  mi_block_t* last = head;
-  mi_block_t* next;
-  while ((next = mi_block_next(page, last)) != NULL && count <= max_count) {
-    count++;
-    last = next;
-  }
-
-  // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free)
-  if mi_unlikely(count > max_count) {
-    _mi_error_message(EFAULT, "corrupted thread-free list\n");
-    return; // the thread-free items cannot be freed
-  }
-  // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list)
-  else if mi_unlikely(count > page->used) {
-    _mi_error_message(EFAULT, "corrupted meta-data in thread-free list\n");
-    return; // the thread-free items cannot be freed
-  }
-
-  // and append the current local free list
-  mi_block_set_next(page, last, page->local_free);
-  page->local_free = head;
-
-  // update counts now
-  mi_assert_internal(count <= UINT16_MAX);
-  mi_assert_internal(page->used >= (uint16_t)count);
-  page->used = page->used - (uint16_t)count;
-}
-
-// Collect the local `thread_free` list using an atomic exchange.
-static void mi_page_thread_free_collect(mi_page_t* page)
-{
-  // atomically capture the thread free list
-  mi_block_t* head;
-  mi_thread_free_t tfreex;
-  mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free);
-  do {
-    head = mi_tf_block(tfree);
-    if mi_likely(head == NULL) return; // return if the list is empty
-    tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree));  // set the thread free list to NULL
-  } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex));  // release is enough?
-  mi_assert_internal(head != NULL);
-
-  // and move it to the local list
-  mi_page_thread_collect_to_local(page, head);
-}
-
-// returns `true` if after collection `mi_page_immediate_available` is true.
-static bool mi_page_free_quick_collect(mi_page_t* page) {
-  if (page->free != NULL) return true;
-  if (page->local_free == NULL) return false;
-  // move local_free to free
-  page->free = page->local_free;
-  page->local_free = NULL;
-  page->free_is_zero = false;
-  return true;
-}
-
-void _mi_page_free_collect(mi_page_t* page, bool force) {
-  mi_assert_internal(page!=NULL);
-
-  // collect the thread free list
-  mi_page_thread_free_collect(page);
-
-  // and the local free list
-  if (page->local_free != NULL) {
-    if mi_likely(page->free == NULL) {
-      // usual case
-      page->free = page->local_free;
-      page->local_free = NULL;
-      page->free_is_zero = false;
-    }
-    else if (force) {
-      // append -- only on shutdown (force) as this is a linear operation
-      mi_block_t* tail = page->local_free;
-      mi_block_t* next;
-      while ((next = mi_block_next(page, tail)) != NULL) {
-        tail = next;
-      }
-      mi_block_set_next(page, tail, page->free);
-      page->free = page->local_free;
-      page->local_free = NULL;
-      page->free_is_zero = false;
-    }
-  }
-
-  mi_assert_internal(!force || page->local_free == NULL);
-}
-
-// Collect elements in the thread-free list starting at `head`. This is an optimized
-// version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`.
-//
-// `head` must be in the `xthread_free` list. It will not collect `head` itself
-// so the `used` count is not fully updated in general. However, if the `head` is
-// the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true).
-void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {
-  if (head == NULL) return;
-  mi_block_t* next = mi_block_next(page,head);  // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops)
-  if (next != NULL) {
-    mi_block_set_next(page, head, NULL);
-    mi_page_thread_collect_to_local(page, next);
-    if (page->local_free != NULL && page->free == NULL) {
-      page->free = page->local_free;
-      page->local_free = NULL;
-      page->free_is_zero = false;
-    }
-  }
-  if (page->used == 1) {
-    // all elements are free'd since we skipped the `head` element itself
-    mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head);
-    mi_assert_internal(mi_block_next(page,head) == NULL);
-    _mi_page_free_collect(page, false);  // collect the final element
-  }
-}
-
-
-/* -----------------------------------------------------------
-  Page fresh and retire
------------------------------------------------------------ */
-
-/*
-// called from segments when reclaiming abandoned pages
-void _mi_page_reclaim(mi_theap_t* theap, mi_page_t* page) {
-  // mi_page_set_theap(page, theap);
-  // _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after theap is set)
-  _mi_page_free_collect(page, false); // ensure used count is up to date
-
-  mi_assert_expensive(mi_page_is_valid_init(page));
-  // mi_assert_internal(mi_page_theap(page) == theap);
-  // mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE);
-
-  // TODO: push on full queue immediately if it is full?
-  mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
-  mi_page_queue_push(theap, pq, page);
-  mi_assert_expensive(_mi_page_is_valid(page));
-}
-*/
-
-// called from `mi_free` on a reclaim, and fresh_alloc if we get an abandoned page
-void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page)
-{
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(mi_page_is_abandoned(page));
-
-  mi_page_set_theap(page,theap);
-  _mi_page_free_collect(page, false); // ensure used count is up to date
-  mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
-  mi_page_queue_push_at_end(theap, pq, page);
-  mi_assert_expensive(_mi_page_is_valid(page));
-}
-
-void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
-  _mi_page_free_collect(page, false); // ensure used count is up to date
-  if (mi_page_all_free(page)) {
-    _mi_page_free(page, pq);
-  }
-  else {
-    mi_page_queue_remove(pq, page);
-    mi_theap_t* theap = page->theap;
-    mi_page_set_theap(page, NULL);
-    page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap
-    _mi_arenas_page_abandon(page, theap);
-    _mi_arenas_collect(false, false, theap->tld); // allow purging
-  }
-}
-
-
-// allocate a fresh page from an arena
-static mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, size_t block_size, size_t page_alignment) {
-  #if !MI_HUGE_PAGE_ABANDON
-  mi_assert_internal(pq != NULL);
-  mi_assert_internal(mi_theap_contains_queue(theap, pq));
-  mi_assert_internal(page_alignment > 0 || block_size > MI_LARGE_MAX_OBJ_SIZE || block_size == pq->block_size);
-  #endif
-  mi_page_t* page = _mi_arenas_page_alloc(theap, block_size, page_alignment);
-  if (page == NULL) {
-    // out-of-memory
-    return NULL;
-  }
-  if (mi_page_is_abandoned(page)) {
-    _mi_theap_page_reclaim(theap, page);
-    if (!mi_page_immediate_available(page)) {
-      if (mi_page_is_expandable(page)) {
-        if (!mi_page_extend_free(theap, page)) {
-          return NULL; // cannot commit
-        };
-      }
-      else {
-        mi_assert(false); // should not happen?
-        return NULL;
-      }
-    }
-  }
-  else if (pq != NULL) {
-    mi_page_queue_push(theap, pq, page);
-  }
-  mi_assert_internal(pq!=NULL || mi_page_block_size(page) >= block_size);
-  mi_assert_expensive(_mi_page_is_valid(page));
-  return page;
-}
-
-// Get a fresh page to use
-static mi_page_t* mi_page_fresh(mi_theap_t* theap, mi_page_queue_t* pq) {
-  mi_assert_internal(mi_theap_contains_queue(theap, pq));
-  mi_page_t* page = mi_page_fresh_alloc(theap, pq, pq->block_size, 0);
-  if (page==NULL) return NULL;
-  mi_assert_internal(pq->block_size==mi_page_block_size(page));
-  mi_assert_internal(pq==mi_theap_page_queue_of(theap, page));
-  return page;
-}
-
-
-/* -----------------------------------------------------------
-  Unfull, abandon, free and retire
------------------------------------------------------------ */
-
-// Move a page from the full list back to a regular list (called from thread-local mi_free)
-void _mi_page_unfull(mi_page_t* page) {
-  mi_assert_internal(page != NULL);
-  mi_assert_expensive(_mi_page_is_valid(page));
-  mi_assert_internal(mi_page_is_in_full(page));
-  mi_assert_internal(!mi_page_theap(page)->allow_page_abandon);
-  if (!mi_page_is_in_full(page)) return;
-
-  mi_theap_t* theap = mi_page_theap(page);
-  mi_page_queue_t* pqfull = &theap->pages[MI_BIN_FULL];
-  mi_page_set_in_full(page, false); // to get the right queue
-  mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
-  mi_page_set_in_full(page, true);
-  mi_page_queue_enqueue_from_full(pq, pqfull, page);
-}
-
-static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
-  mi_assert_internal(pq == mi_page_queue_of(page));
-  mi_assert_internal(!mi_page_immediate_available(page));
-  mi_assert_internal(!mi_page_is_in_full(page));
-
-  mi_theap_t* theap = mi_page_theap(page);
-  if (theap->allow_page_abandon) {
-    // abandon full pages (this is the usual case in order to allow for sharing of memory between theaps)
-    _mi_page_abandon(page, pq);
-  }
-  else if (!mi_page_is_in_full(page)) {
-    // put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed)
-    mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page);
-    _mi_page_free_collect(page, false);  // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set
-  }
-}
-
-
-// Free a page with no more free blocks
-void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) {
-  mi_assert_internal(page != NULL);
-  mi_assert_expensive(_mi_page_is_valid(page));
-  mi_assert_internal(pq == mi_page_queue_of(page));
-  mi_assert_internal(mi_page_all_free(page));
-  // mi_assert_internal(mi_page_thread_free_flag(page)!=MI_DELAYED_FREEING);
-
-  // no more aligned blocks in here
-  mi_page_set_has_interior_pointers(page, false);
-
-  // remove from the page list
-  // (no need to do _mi_theap_delayed_free first as all blocks are already free)
-  mi_page_queue_remove(pq, page);
-
-  // and free it
-  mi_theap_t* theap = mi_page_theap(page); mi_assert_internal(theap!=NULL);
-  mi_page_set_theap(page,NULL);
-  _mi_arenas_page_free(page, theap);
-  _mi_arenas_collect(false, false, theap->tld);  // allow purging
-}
-
-#define MI_MAX_RETIRE_SIZE    MI_LARGE_OBJ_SIZE_MAX   // should be less than size for MI_BIN_HUGE
-#define MI_RETIRE_CYCLES      (16)
-
-// Retire a page with no more used blocks
-// Important to not retire too quickly though as new
-// allocations might coming.
-//
-// Note: called from `mi_free` and benchmarks often
-// trigger this due to freeing everything and then
-// allocating again so careful when changing this.
-void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {
-  mi_assert_internal(page != NULL);
-  mi_assert_expensive(_mi_page_is_valid(page));
-  mi_assert_internal(mi_page_all_free(page));
-
-  if (page->retire_expire!=0) return;  // already retired, just keep it retired
-  mi_page_set_has_interior_pointers(page, false);
-
-  // don't retire too often..
-  // (or we end up retiring and re-allocating most of the time)
-  // NOTE: refine this more: we should not retire if this
-  // is the only page left with free blocks. It is not clear
-  // how to check this efficiently though...
-  // for now, we don't retire if it is the only page left of this size class.
-  mi_page_queue_t* pq = mi_page_queue_of(page);
-  #if MI_RETIRE_CYCLES > 0
-  const size_t bsize = mi_page_block_size(page);
-  if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) {  // not full or huge queue?
-    if (pq->last==page && pq->first==page) { // the only page in the queue?
-      mi_theap_t* theap = mi_page_theap(page);
-      #if MI_STAT>0
-      mi_theap_stat_counter_increase(theap, pages_retire, 1);
-      #endif
-      page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4);
-      mi_assert_internal(pq >= theap->pages);
-      const size_t index = pq - theap->pages;
-      mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE);
-      if (index < theap->page_retired_min) theap->page_retired_min = index;
-      if (index > theap->page_retired_max) theap->page_retired_max = index;
-      mi_assert_internal(mi_page_all_free(page));
-      return; // don't free after all
-    }
-  }
-  #endif
-  _mi_page_free(page, pq);
-}
-
-// free retired pages: we don't need to look at the entire queues
-// since we only retire pages that are at the head position in a queue.
-void _mi_theap_collect_retired(mi_theap_t* theap, bool force) {
-  size_t min = MI_BIN_FULL;
-  size_t max = 0;
-  for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) {
-    mi_page_queue_t* pq   = &theap->pages[bin];
-    mi_page_t*       page = pq->first;
-    if (page != NULL && page->retire_expire != 0) {
-      if (mi_page_all_free(page)) {
-        page->retire_expire--;
-        if (page->retire_expire == 0 || force) {
-          _mi_page_free(page, pq);
-        }
-        else {
-          // keep retired, update min/max
-          if (bin < min) min = bin;
-          if (bin > max) max = bin;
-        }
-      }
-      else {
-        page->retire_expire = 0;
-      }
-    }
-  }
-  theap->page_retired_min = min;
-  theap->page_retired_max = max;
-}
-
-/*
-static void mi_theap_collect_full_pages(mi_theap_t* theap) {
-  // note: normally full pages get immediately abandoned and the full queue is always empty
-  // this path is only used if abandoning is disabled due to a destroy-able theap or options
-  // set by the user.
-  mi_page_queue_t* pq = &theap->pages[MI_BIN_FULL];
-  for (mi_page_t* page = pq->first; page != NULL; ) {
-    mi_page_t* next = page->next;         // get next in case we free the page
-    _mi_page_free_collect(page, false);   // register concurrent free's
-    // no longer full?
-    if (!mi_page_is_full(page)) {
-      if (mi_page_all_free(page)) {
-        _mi_page_free(page, pq);
-      }
-      else {
-        _mi_page_unfull(page);
-      }
-    }
-    page = next;
-  }
-}
-*/
-
-
-/* -----------------------------------------------------------
-  Initialize the initial free list in a page.
-  In secure mode we initialize a randomized list by
-  alternating between slices.
------------------------------------------------------------ */
-
-#define MI_MAX_SLICE_SHIFT  (6)   // at most 64 slices
-#define MI_MAX_SLICES       (1UL << MI_MAX_SLICE_SHIFT)
-#define MI_MIN_SLICES       (2)
-
-static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* const page, const size_t bsize, const size_t extend) {
-  #if (MI_SECURE<3)
-  mi_assert_internal(page->free == NULL);
-  mi_assert_internal(page->local_free == NULL);
-  #endif
-  mi_assert_internal(page->capacity + extend <= page->reserved);
-  mi_assert_internal(bsize == mi_page_block_size(page));
-  void* const page_area = mi_page_start(page);
-
-  // initialize a randomized free list
-  // set up `slice_count` slices to alternate between
-  size_t shift = MI_MAX_SLICE_SHIFT;
-  while ((extend >> shift) == 0) {
-    shift--;
-  }
-  const size_t slice_count = (size_t)1U << shift;
-  const size_t slice_extend = extend / slice_count;
-  mi_assert_internal(slice_extend >= 1);
-  mi_block_t* blocks[MI_MAX_SLICES];   // current start of the slice
-  size_t      counts[MI_MAX_SLICES];   // available objects in the slice
-  for (size_t i = 0; i < slice_count; i++) {
-    blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i*slice_extend);
-    counts[i] = slice_extend;
-  }
-  counts[slice_count-1] += (extend % slice_count);  // final slice holds the modulus too (todo: distribute evenly?)
-
-  // and initialize the free list by randomly threading through them
-  // set up first element
-  const uintptr_t r = _mi_theap_random_next(theap);
-  size_t current = r % slice_count;
-  counts[current]--;
-  mi_block_t* const free_start = blocks[current];
-  // and iterate through the rest; use `random_shuffle` for performance
-  uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0
-  for (size_t i = 1; i < extend; i++) {
-    // call random_shuffle only every INTPTR_SIZE rounds
-    const size_t round = i%MI_INTPTR_SIZE;
-    if (round == 0) rnd = _mi_random_shuffle(rnd);
-    // select a random next slice index
-    size_t next = ((rnd >> 8*round) & (slice_count-1));
-    while (counts[next]==0) {                            // ensure it still has space
-      next++;
-      if (next==slice_count) next = 0;
-    }
-    // and link the current block to it
-    counts[next]--;
-    mi_block_t* const block = blocks[current];
-    blocks[current] = (mi_block_t*)((uint8_t*)block + bsize);  // bump to the following block
-    mi_block_set_next(page, block, blocks[next]);   // and set next; note: we may have `current == next`
-    current = next;
-  }
-  // prepend to the free list (usually NULL)
-  mi_block_set_next(page, blocks[current], page->free);  // end of the list
-  page->free = free_start;
-}
-
-static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend)
-{
-  #if (MI_SECURE<3)
-  mi_assert_internal(page->free == NULL);
-  mi_assert_internal(page->local_free == NULL);
-  #endif
-  mi_assert_internal(page->capacity + extend <= page->reserved);
-  mi_assert_internal(bsize == mi_page_block_size(page));
-  void* const page_area = mi_page_start(page);
-
-  mi_block_t* const start = mi_page_block_at(page, page_area, bsize, page->capacity);
-
-  // initialize a sequential free list
-  mi_block_t* const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1);
-  mi_block_t* block = start;
-  while(block <= last) {
-    mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize);
-    mi_block_set_next(page,block,next);
-    block = next;
-  }
-  // prepend to free list (usually `NULL`)
-  mi_block_set_next(page, last, page->free);
-  page->free = start;
-}
-
-/* -----------------------------------------------------------
-  Page initialize and extend the capacity
------------------------------------------------------------ */
-
-#define MI_MAX_EXTEND_SIZE    (4*1024)      // heuristic, one OS page seems to work well.
-#if (MI_SECURE>=2)
-#define MI_MIN_EXTEND         (8*MI_SECURE) // extend at least by this many
-#else
-#define MI_MIN_EXTEND         (1)
-#endif
-
-// Extend the capacity (up to reserved) by initializing a free list
-// We do at most `MI_MAX_EXTEND` to avoid touching too much memory
-// Note: we also experimented with "bump" allocation on the first
-// allocations but this did not speed up any benchmark (due to an
-// extra test in malloc? or cache effects?)
-static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {
-  mi_assert_expensive(mi_page_is_valid_init(page));
-  #if (MI_SECURE<3)
-  mi_assert(page->free == NULL);
-  mi_assert(page->local_free == NULL);
-  if (page->free != NULL) return true;
-  #endif
-  if (page->capacity >= page->reserved) return true;
-
-  size_t page_size;
-  //uint8_t* page_start =
-  mi_page_area(page, &page_size);
-  #if MI_STAT>0
-  mi_theap_stat_counter_increase(theap, pages_extended, 1);
-  #endif
-
-  // calculate the extend count
-  const size_t bsize = mi_page_block_size(page);
-  size_t extend = (size_t)page->reserved - page->capacity;
-  mi_assert_internal(extend > 0);
-
-  size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/bsize);
-  if (max_extend < MI_MIN_EXTEND) { max_extend = MI_MIN_EXTEND; }
-  mi_assert_internal(max_extend > 0);
-
-  if (extend > max_extend) {
-    // ensure we don't touch memory beyond the page to reduce page commit.
-    // the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%.
-    extend = max_extend;
-  }
-
-  mi_assert_internal(extend > 0 && extend + page->capacity <= page->reserved);
-  mi_assert_internal(extend < (1UL<<16));
-
-  // commit on demand?
-  if (page->slice_committed > 0) {
-    const size_t needed_size = (page->capacity + extend)*bsize;
-    const size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), MI_PAGE_MIN_COMMIT_SIZE );
-    if (needed_commit > page->slice_committed) {
-      mi_assert_internal(((needed_commit - page->slice_committed) % _mi_os_page_size()) == 0);
-      if (!_mi_os_commit(mi_page_slice_start(page) + page->slice_committed, needed_commit - page->slice_committed, NULL)) {
-        return false;
-      }
-      page->slice_committed = needed_commit;
-    }
-  }
-
-  // and append the extend the free list
-  if (extend < MI_MIN_SLICES || MI_SECURE<2) { //!mi_option_is_enabled(mi_option_secure)) {
-    mi_page_free_list_extend(page, bsize, extend );
-  }
-  else {
-    mi_page_free_list_extend_secure(theap, page, bsize, extend);
-  }
-  // enable the new free list
-  page->capacity += (uint16_t)extend;
-  #if MI_STAT>0
-  mi_theap_stat_increase(theap, page_committed, extend * bsize);
-  #endif
-  mi_assert_expensive(mi_page_is_valid_init(page));
-  return true;
-}
-
-// Initialize a fresh page (that is already partially initialized)
-mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {
-  mi_assert(page != NULL);
-  mi_assert(theap!=NULL);
-  page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap`
-  mi_page_set_theap(page, theap);
-
-  size_t page_size;
-  uint8_t* page_start = mi_page_area(page, &page_size); MI_UNUSED(page_start);
-  mi_track_mem_noaccess(page_start,page_size);
-  mi_assert_internal(page_size / mi_page_block_size(page) < (1L<<16));
-  mi_assert_internal(page->reserved > 0);
-  #if (MI_PADDING || MI_ENCODE_FREELIST)
-  page->keys[0] = _mi_theap_random_next(theap);
-  page->keys[1] = _mi_theap_random_next(theap);
-  #endif
-  #if MI_DEBUG>2
-  if (page->memid.initially_zero) {
-    mi_track_mem_defined(page->page_start, mi_page_committed(page));
-    mi_assert_expensive(mi_mem_is_zero(page_start, mi_page_committed(page)));
-  }
-  #endif
-
-  mi_assert_internal(page->theap!=NULL);
-  mi_assert_internal(page->theap == mi_page_theap(page));
-  mi_assert_internal(page->capacity == 0);
-  mi_assert_internal(page->free == NULL);
-  mi_assert_internal(page->used == 0);
-  mi_assert_internal(mi_page_is_owned(page));
-  mi_assert_internal(page->xthread_free == 1);
-  mi_assert_internal(page->next == NULL);
-  mi_assert_internal(page->prev == NULL);
-  mi_assert_internal(page->retire_expire == 0);
-  mi_assert_internal(!mi_page_has_interior_pointers(page));
-  #if (MI_PADDING || MI_ENCODE_FREELIST)
-  mi_assert_internal(page->keys[0] != 0);
-  mi_assert_internal(page->keys[1] != 0);
-  #endif
-  mi_assert_expensive(mi_page_is_valid_init(page));
-
-  // initialize an initial free list
-  if (!mi_page_extend_free(theap,page)) return false;
-  mi_assert(mi_page_immediate_available(page));
-  return true;
-}
-
-
-/* -----------------------------------------------------------
-  Find pages with free blocks
--------------------------------------------------------------*/
-
-// Find a page with free blocks of `page->block_size`.
-static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_page_queue_t* pq, bool first_try)
-{
-  // search through the pages in "next fit" order
-  size_t count = 0;
-  long candidate_limit = 0;          // we reset this on the first candidate to limit the search
-  long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages
-  mi_page_t* page_candidate = NULL;  // a page with free space
-  mi_page_t* page = pq->first;
-
-  while (page != NULL)
-  {
-    mi_page_t* next = page->next; // remember next (as this page can move to another queue)
-    count++;
-    candidate_limit--;
-
-    // search up to N pages for a best candidate
-
-    // is the local free list non-empty?
-    bool immediate_available = mi_page_immediate_available(page);
-    if (!immediate_available) {
-      // collect freed blocks by us and other threads to we get a proper use count
-      _mi_page_free_collect(page, false);
-      immediate_available = mi_page_immediate_available(page);
-    }
-
-    // if the page is completely full, move it to the `mi_pages_full`
-    // queue so we don't visit long-lived pages too often.
-    if (!immediate_available && !mi_page_is_expandable(page)) {
-      page_full_retain--;
-      if (page_full_retain < 0) {
-        mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));
-        mi_page_to_full(page, pq);
-      }
-    }
-    else {
-      // the page has free space, make it a candidate
-      // we prefer non-expandable pages with high usage as candidates (to reduce commit, and increase chances of free-ing up pages)
-      if (page_candidate == NULL) {
-        page_candidate = page;
-        candidate_limit = _mi_option_get_fast(mi_option_page_max_candidates);
-      }
-      else if (mi_page_all_free(page_candidate)) {
-        _mi_page_free(page_candidate, pq);
-        page_candidate = page;
-      }
-      // prefer to reuse fuller pages (in the hope the less used page gets freed)
-      else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) {
-        page_candidate = page;
-      }
-      // if we find a non-expandable candidate, or searched for N pages, return with the best candidate
-      if (immediate_available || candidate_limit <= 0) {
-        mi_assert_internal(page_candidate!=NULL);
-        break;
-      }
-    }
-
-  #if 0
-    // first-fit algorithm without candidates
-    // If the page contains free blocks, we are done
-    if (mi_page_immediate_available(page) || mi_page_is_expandable(page)) {
-      break;  // pick this one
-    }
-
-    // If the page is completely full, move it to the `mi_pages_full`
-    // queue so we don't visit long-lived pages too often.
-    mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));
-    mi_page_to_full(page, pq);
-  #endif
-
-    page = next;
-  } // for each page
-
-  mi_theap_stat_counter_increase(theap, page_searches, count);
-  mi_theap_stat_counter_increase(theap, page_searches_count, 1);
-
-  // set the page to the best candidate
-  if (page_candidate != NULL) {
-    page = page_candidate;
-  }
-  if (page != NULL) {
-    if (!mi_page_immediate_available(page)) {
-      mi_assert_internal(mi_page_is_expandable(page));
-      if (!mi_page_extend_free(theap, page)) {
-        page = NULL; // failed to extend
-      }
-    }
-    mi_assert_internal(page == NULL || mi_page_immediate_available(page));
-  }
-
-  if (page == NULL) {
-    _mi_theap_collect_retired(theap, false); // perhaps make a page available
-    page = mi_page_fresh(theap, pq);
-    mi_assert_internal(page == NULL || mi_page_immediate_available(page));
-    if (page == NULL && first_try) {
-      // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again
-      page = mi_page_queue_find_free_ex(theap, pq, false);
-      mi_assert_internal(page == NULL || mi_page_immediate_available(page));
-    }
-  }
-  else {
-    mi_assert_internal(page == NULL || mi_page_immediate_available(page));
-    // move the page to the front of the queue
-    mi_page_queue_move_to_front(theap, pq, page);
-    page->retire_expire = 0;
-    // _mi_theap_collect_retired(theap, false); // update retire counts; note: increases rss on MemoryLoad bench so don't do this
-  }
-  mi_assert_internal(page == NULL || mi_page_immediate_available(page));
-
-
-  return page;
-}
-
-
-
-// Find a page with free blocks of `size`.
-static mi_page_t* mi_find_free_page(mi_theap_t* theap, mi_page_queue_t* pq) {
-  // mi_page_queue_t* pq = mi_page_queue(theap, size);
-  mi_assert_internal(!mi_page_queue_is_huge(pq));
-
-  // check the first page: we even do this with candidate search or otherwise we re-search every time
-  mi_page_t* page = pq->first;
-  if mi_likely(page != NULL && mi_page_free_quick_collect(page)) {
-    #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness
-    if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) {
-      (void)mi_page_extend_free(theap, page);  // ok if this fails
-      mi_assert_internal(mi_page_immediate_available(page));
-    }
-    #endif
-    page->retire_expire = 0;
-    return page; // fast path
-  }
-  else {
-    return mi_page_queue_find_free_ex(theap, pq, true);
-  }
-}
-
-
-/* -----------------------------------------------------------
-  Users can register a deferred free function called
-  when the `free` list is empty. Since the `local_free`
-  is separate this is deterministically called after
-  a certain number of allocations.
------------------------------------------------------------ */
-
-// The program should only install a single deferred free handler before doing allocation.
-static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers)
-static _Atomic(void*) deferred_arg;   
-
-void _mi_deferred_free(mi_theap_t* theap, bool force) {
-  theap->heartbeat++;
-  mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free);
-  if (fun != NULL && !theap->tld->recurse) {
-    theap->tld->recurse = true;
-    void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg);  
-    fun(force, theap->heartbeat, arg);
-    theap->tld->recurse = false;
-  }
-}
-
-void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {
-  mi_atomic_store_ptr_release(void,&deferred_arg, arg);
-  mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn);  
-}
-
-
-/* -----------------------------------------------------------
-  General allocation
------------------------------------------------------------ */
-
-// Huge pages contain just one block, and the segment contains just that page.
-// Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX)
-// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`.
-static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page_alignment, mi_page_queue_t* pq) {
-  const size_t block_size = _mi_os_good_alloc_size(size);
-  // mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0);
-  #if MI_HUGE_PAGE_ABANDON
-  #error todo.
-  #else
-  // mi_page_queue_t* pq = mi_page_queue(theap, MI_LARGE_MAX_OBJ_SIZE+1);  // always in the huge queue regardless of the block size
-  mi_assert_internal(mi_page_queue_is_huge(pq));
-  #endif
-  mi_page_t* page = mi_page_fresh_alloc(theap, pq, block_size, page_alignment);
-  if (page != NULL) {
-    mi_assert_internal(mi_page_block_size(page) >= size);
-    mi_assert_internal(mi_page_immediate_available(page));
-    mi_assert_internal(mi_page_is_huge(page));
-    mi_assert_internal(mi_page_is_singleton(page));
-    #if MI_HUGE_PAGE_ABANDON
-    mi_assert_internal(mi_page_is_abandoned(page));
-    mi_page_set_theap(page, NULL);
-    #endif
-    mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page));
-    mi_theap_stat_counter_increase(theap, malloc_huge_count, 1);
-  }
-  return page;
-}
-
-
-// Allocate a page
-// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
-static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept {
-  const size_t req_size = size - MI_PADDING_SIZE;  // correct for padding_size in case of an overflow on `size`
-  if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) {
-    _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size);
-    return NULL;
-  }
-  mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size));
-  // huge allocation?
-  if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) {
-    return mi_huge_page_alloc(theap,size,huge_alignment,pq);
-  }
-  else {
-    // otherwise find a page with free blocks in our size segregated queues
-    #if MI_PADDING
-    mi_assert_internal(size >= MI_PADDING_SIZE);
-    #endif
-    return mi_find_free_page(theap, pq);
-  }
-}
-
-
-// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.
-// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
-// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for
-// very large requested alignments in which case we use a huge singleton page.
-// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit)
-// to use 4 parameters which compiles better on msvc for the malloc fast path.
-void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept
-{
-  const bool zero = ((zero_huge_alignment & 1) != 0);
-  const size_t huge_alignment = (zero_huge_alignment & ~1);
-
-  #if !MI_THEAP_INITASNULL
-  mi_assert_internal(theap != NULL);
-  #endif
-
-  // initialize if necessary
-  if mi_unlikely(!mi_theap_is_initialized(theap)) {
-    if (theap==&_mi_theap_empty_wrong) {
-      // we were unable to allocate a theap for a first-class heap
-      return NULL;
-    }
-    // otherwise we initialize the thread and its default theap
-    mi_thread_init();
-    theap = _mi_theap_default();
-    if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; }
-    mi_assert_internal(_mi_theap_default()==theap);
-  }
-  mi_assert_internal(mi_theap_is_initialized(theap));
-
-  // do administrative tasks every N generic mallocs
-  if mi_unlikely(++theap->generic_count >= 1000) {
-    theap->generic_collect_count += theap->generic_count;
-    theap->generic_count = 0;
-    // call potential deferred free routines
-    _mi_deferred_free(theap, false);
-    // free retired pages
-    _mi_theap_collect_retired(theap, false);
-
-    // collect every once in a while (10000 by default)
-    const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L);
-    if (theap->generic_collect_count >= generic_collect) {
-      theap->generic_collect_count = 0;
-      mi_theap_collect(theap, false /* force? */);
-    }
-  }
-
-  // find (or allocate) a page of the right size
-  mi_page_t* page = mi_find_page(theap, size, huge_alignment);
-  if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more
-    mi_theap_collect(theap, true /* force? */);
-    page = mi_find_page(theap, size, huge_alignment);
-  }
-
-  if mi_unlikely(page == NULL) { // out of memory
-    const size_t req_size = size - MI_PADDING_SIZE;  // correct for padding_size in case of an overflow on `size`
-    _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size);
-    return NULL;
-  }
-
-  mi_assert_internal(mi_page_immediate_available(page));
-  mi_assert_internal(mi_page_block_size(page) >= size);
-  mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
-  mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
-
-  // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc)
-  if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
-  void* const p = _mi_page_malloc_zero(theap,page,size,zero);
-  mi_assert_internal(p != NULL);
-
-  // move full pages to the full queue
-  if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) {
-    mi_page_to_full(page, mi_page_queue_of(page));
-  }
-  return p;
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/emscripten/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/emscripten/prim.c
deleted file mode 100644
index 1ba0936e..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/emscripten/prim.c
+++ /dev/null
@@ -1,261 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen, Alon Zakai
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-// This file is included in `src/prim/prim.c`
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/atomic.h"
-#include "mimalloc/prim.h"
-
-#include <sched.h>   // sched_yield
-#include <unistd.h>  // getentropy
-
-// Design
-// ======
-//
-// mimalloc is built on top of emmalloc. emmalloc is a minimal allocator on top
-// of sbrk. The reason for having three layers here is that we want mimalloc to
-// be able to allocate and release system memory properly, the same way it would
-// when using VirtualAlloc on Windows or mmap on POSIX, and sbrk is too limited.
-// Specifically, sbrk can only go up and down, and not "skip" over regions, and
-// so we end up either never freeing memory to the system, or we can get stuck
-// with holes.
-//
-// Atm wasm generally does *not* free memory back the system: once grown, we do
-// not shrink back down (https://github.com/WebAssembly/design/issues/1397).
-// However, that is expected to improve
-// (https://github.com/WebAssembly/memory-control/blob/main/proposals/memory-control/Overview.md)
-// and so we do not want to bake those limitations in here.
-//
-// Even without that issue, we want our system allocator to handle holes, that
-// is, it should merge freed regions and allow allocating new content there of
-// the full size, etc., so that we do not waste space. That means that the
-// system allocator really does need to handle the general problem of allocating
-// and freeing variable-sized chunks of memory in a random order, like malloc/
-// free do. And so it makes sense to layer mimalloc on top of such an
-// implementation.
-//
-// emmalloc makes sense for the lower level because it is small and simple while
-// still fully handling merging of holes etc. It is not the most efficient
-// allocator, but our assumption is that mimalloc needs to be fast while the
-// system allocator underneath it is called much less frequently.
-//
-
-//---------------------------------------------
-// init
-//---------------------------------------------
-
-void _mi_prim_mem_init( mi_os_mem_config_t* config) {
-  config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
-  config->alloc_granularity = 16;
-  config->has_overcommit = false;
-  config->has_partial_free = false;
-  config->has_virtual_reserve = false;
-}
-
-extern void emmalloc_free(void*);
-
-int _mi_prim_free(void* addr, size_t size) {
-  if (size==0) return 0;
-  emmalloc_free(addr);
-  return 0;
-}
-
-
-//---------------------------------------------
-// Allocation
-//---------------------------------------------
-
-extern void* emmalloc_memalign(size_t alignment, size_t size);
-
-// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
-int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
-  MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);
-  *is_large = false;
-  // TODO: Track the highest address ever seen; first uses of it are zeroes.
-  //       That assumes no one else uses sbrk but us (they could go up,
-  //       scribble, and then down), but we could assert on that perhaps.
-  *is_zero = false;
-  // emmalloc has a minimum alignment size.
-  #define MIN_EMMALLOC_ALIGN           8
-  if (try_alignment < MIN_EMMALLOC_ALIGN) {
-    try_alignment = MIN_EMMALLOC_ALIGN;
-  }
-  void* p = emmalloc_memalign(try_alignment, size);
-  *addr = p;
-  if (p == 0) {
-    return ENOMEM;
-  }
-  return 0;
-}
-
-
-//---------------------------------------------
-// Commit/Reset
-//---------------------------------------------
-
-int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  // See TODO above.
-  *is_zero = false;
-  return 0;
-}
-
-int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  *needs_recommit = false;
-  return 0;
-}
-
-int _mi_prim_reset(void* addr, size_t size) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  return 0;
-}
-
-int _mi_prim_reuse(void* addr, size_t size) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  return 0;
-}
-
-int _mi_prim_protect(void* addr, size_t size, bool protect) {
-  MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);
-  return 0;
-}
-
-
-//---------------------------------------------
-// Huge pages and NUMA nodes
-//---------------------------------------------
-
-int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
-  MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
-  *is_zero = true;
-  *addr = NULL;
-  return ENOSYS;
-}
-
-size_t _mi_prim_numa_node(void) {
-  return 0;
-}
-
-size_t _mi_prim_numa_node_count(void) {
-  return 1;
-}
-
-
-//----------------------------------------------------------------
-// Clock
-//----------------------------------------------------------------
-
-#include <emscripten/html5.h>
-
-mi_msecs_t _mi_prim_clock_now(void) {
-  return emscripten_date_now();
-}
-
-
-//----------------------------------------------------------------
-// Process info
-//----------------------------------------------------------------
-
-void _mi_prim_process_info(mi_process_info_t* pinfo)
-{
-  // use defaults
-  MI_UNUSED(pinfo);
-}
-
-
-//----------------------------------------------------------------
-// Output
-//----------------------------------------------------------------
-
-#include <emscripten/console.h>
-
-void _mi_prim_out_stderr( const char* msg) {
-  emscripten_console_error(msg);
-}
-
-
-//----------------------------------------------------------------
-// Environment
-//----------------------------------------------------------------
-
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
-  // For code size reasons, do not support environ customization for now.
-  MI_UNUSED(name);
-  MI_UNUSED(result);
-  MI_UNUSED(result_size);
-  return false;
-}
-
-
-//----------------------------------------------------------------
-// Random
-//----------------------------------------------------------------
-
-bool _mi_prim_random_buf(void* buf, size_t buf_len) {
-  int err = getentropy(buf, buf_len);
-  return !err;
-}
-
-
-//----------------------------------------------------------------
-// Thread init/done
-//----------------------------------------------------------------
-
-#if defined(MI_USE_PTHREADS)
-
-// use pthread local storage keys to detect thread ending
-// (and used with MI_TLS_PTHREADS for the default theap)
-pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
-
-static void mi_pthread_done(void* value) {
-  if (value!=NULL) {
-    _mi_thread_done((mi_theap_t*)value);
-  }
-}
-
-void _mi_prim_thread_init_auto_done(void) {
-  mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));
-  pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);
-}
-
-void _mi_prim_thread_done_auto_done(void) {
-  if (_mi_heap_default_key != (pthread_key_t)(-1)) {  // do not leak the key, see issue #809
-    pthread_key_delete(_mi_heap_default_key);
-  }
-}
-
-void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-  if (_mi_heap_default_key != (pthread_key_t)(-1)) {  // can happen during recursive invocation on freeBSD
-    pthread_setspecific(_mi_heap_default_key, theap);
-  }
-}
-
-#else
-
-void _mi_prim_thread_init_auto_done(void) {
-  // nothing
-}
-
-void _mi_prim_thread_done_auto_done(void) {
-  // nothing
-}
-
-void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-  MI_UNUSED(theap);
-}
-#endif
-
-bool _mi_prim_thread_is_in_threadpool(void) {
-  return false;
-}
-
-void _mi_prim_thread_yield(void) {
-  sched_yield();
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/alloc-override-zone.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/alloc-override-zone.c
deleted file mode 100644
index fdbe095c..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/alloc-override-zone.c
+++ /dev/null
@@ -1,481 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-
-#if defined(MI_MALLOC_OVERRIDE)
-
-#if !defined(__APPLE__)
-#error "this file should only be included on macOS"
-#endif
-
-/* ------------------------------------------------------
-   Override system malloc on macOS
-   This is done through the malloc zone interface.
-   It seems to be most robust in combination with interposing
-   though or otherwise we may get zone errors as there are could
-   be allocations done by the time we take over the
-   zone.
------------------------------------------------------- */
-
-#include <AvailabilityMacros.h>
-#include <malloc/malloc.h>
-#include <string.h>  // memset
-#include <stdlib.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
-// only available from OSX 10.6
-extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import));
-#endif
-
-/* ------------------------------------------------------
-   malloc zone members
------------------------------------------------------- */
-
-static bool is_mimalloc_zone( malloc_zone_t* zone ); 
-
-static size_t zone_size(malloc_zone_t* zone, const void* p) {
-  if (mi_any_heap_contains(p)) { 
-    return mi_usable_size(p);
-  }
-  else if (!is_mimalloc_zone(zone)) {  // can happen due to interpose
-    return zone->size(zone,p);
-  }
-  else {
-    return 0;
-  }
-}
-
-static void* zone_malloc(malloc_zone_t* zone, size_t size) {
-  MI_UNUSED(zone);
-  return mi_malloc(size);
-}
-
-static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) {
-  MI_UNUSED(zone);
-  return mi_calloc(count, size);
-}
-
-static void* zone_valloc(malloc_zone_t* zone, size_t size) {
-  MI_UNUSED(zone);
-  return mi_malloc_aligned(size, _mi_os_page_size());
-}
-
-static void zone_free(malloc_zone_t* zone, void* p) {
-  if (mi_any_heap_contains(p)) {
-    mi_free(p); // with the page_map and pagemap_commit=1 we can use the regular free
-  }
-  else if (!is_mimalloc_zone(zone)) {  // can happen due to interpose
-    zone->free(zone,p);
-  }
-}
-
-static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) {
-  if (p == NULL || mi_any_heap_contains(p)) {
-    return mi_realloc(p, newsize);
-  }
-  else if (!is_mimalloc_zone(zone)) {  // can happen due to interpose
-    return zone->realloc(zone,p,newsize);
-  }
-  else {
-    return NULL;
-  }
-}
-
-static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) {
-  MI_UNUSED(zone);
-  return mi_malloc_aligned(size,alignment);
-}
-
-static void zone_destroy(malloc_zone_t* zone) {
-  if (!is_mimalloc_zone(zone)) {
-    zone->destroy(zone);
-  }
-}
-
-static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) {
-  unsigned i;
-  for (i = 0; i < count; i++) {
-    ps[i] = zone_malloc(zone, size);
-    if (ps[i] == NULL) break;
-  }
-  return i;
-}
-
-static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) {
-  for(size_t i = 0; i < count; i++) {
-    zone_free(zone, ps[i]);
-    ps[i] = NULL;
-  }
-}
-
-static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) {
-  MI_UNUSED(zone); MI_UNUSED(size);
-  mi_collect(false);
-  return 0;
-}
-
-static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) {
-  MI_UNUSED(size);
-  zone_free(zone,p);
-}
-
-static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) {
-  MI_UNUSED(zone);
-  return mi_is_in_heap_region(p);
-}
-
-
-/* ------------------------------------------------------
-   Introspection members
------------------------------------------------------- */
-
-static kern_return_t intro_enumerator(task_t task, void* p,
-                            unsigned type_mask, vm_address_t zone_address,
-                            memory_reader_t reader,
-                            vm_range_recorder_t recorder)
-{
-  // todo: enumerate all memory
-  MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address);
-  MI_UNUSED(reader); MI_UNUSED(recorder);
-  return KERN_SUCCESS;
-}
-
-static size_t intro_good_size(malloc_zone_t* zone, size_t size) {
-  MI_UNUSED(zone);
-  return mi_good_size(size);
-}
-
-static boolean_t intro_check(malloc_zone_t* zone) {
-  MI_UNUSED(zone);
-  return true;
-}
-
-static void intro_print(malloc_zone_t* zone, boolean_t verbose) {
-  MI_UNUSED(zone); MI_UNUSED(verbose);
-  mi_stats_print(NULL);
-}
-
-static void intro_log(malloc_zone_t* zone, void* p) {
-  MI_UNUSED(zone); MI_UNUSED(p);
-  // todo?
-}
-
-static void intro_force_lock(malloc_zone_t* zone) {
-  MI_UNUSED(zone);
-  // todo?
-}
-
-static void intro_force_unlock(malloc_zone_t* zone) {
-  MI_UNUSED(zone);
-  // todo?
-}
-
-static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) {
-  MI_UNUSED(zone);
-  // todo...
-  stats->blocks_in_use = 0;
-  stats->size_in_use = 0;
-  stats->max_size_in_use = 0;
-  stats->size_allocated = 0;
-}
-
-static boolean_t intro_zone_locked(malloc_zone_t* zone) {
-  MI_UNUSED(zone);
-  return false;
-}
-
-
-/* ------------------------------------------------------
-  At process start, override the default allocator
------------------------------------------------------- */
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
-#endif
-
-#if defined(__clang__)
-#pragma clang diagnostic ignored "-Wc99-extensions"
-#endif
-
-static malloc_introspection_t mi_introspect = {
-  .enumerator = &intro_enumerator,
-  .good_size = &intro_good_size,
-  .check = &intro_check,
-  .print = &intro_print,
-  .log = &intro_log,
-  .force_lock = &intro_force_lock,
-  .force_unlock = &intro_force_unlock,
-#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)
-  .statistics = &intro_statistics,
-  .zone_locked = &intro_zone_locked,
-#endif
-};
-
-static malloc_zone_t mi_malloc_zone = {
-  // note: even with designators, the order is important for C++ compilation
-  //.reserved1 = NULL,
-  //.reserved2 = NULL,
-  .size = &zone_size,
-  .malloc = &zone_malloc,
-  .calloc = &zone_calloc,
-  .valloc = &zone_valloc,
-  .free = &zone_free,
-  .realloc = &zone_realloc,
-  .destroy = &zone_destroy,
-  .zone_name = "mimalloc",
-  .batch_malloc = &zone_batch_malloc,
-  .batch_free = &zone_batch_free,
-  .introspect = &mi_introspect,
-#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)
-  #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)
-  .version = 10,
-  #else
-  .version = 9,
-  #endif
-  // switch to version 9+ on OSX 10.6 to support memalign.
-  .memalign = &zone_memalign,
-  .free_definite_size = &zone_free_definite_size,
-  #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
-  .pressure_relief = &zone_pressure_relief,
-  #endif
-  #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)
-  .claimed_address = &zone_claimed_address,
-  #endif
-#else
-  .version = 4,
-#endif
-};
-
-#ifdef __cplusplus
-}
-#endif
-
-static bool is_mimalloc_zone( malloc_zone_t* zone ) {
-  return (zone==NULL || zone==&mi_malloc_zone);
-}
-
-#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT)
-
-// ------------------------------------------------------
-// Override malloc_xxx and malloc_zone_xxx api's to use only
-// our mimalloc zone. Since even the loader uses malloc
-// on macOS, this ensures that all allocations go through
-// mimalloc (as all calls are interposed).
-// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`,
-// Here, we also override macOS specific API's like
-// `malloc_zone_calloc` etc. see <https://github.com/aosm/libmalloc/blob/master/man/malloc_zone_malloc.3>
-// ------------------------------------------------------
-
-static inline malloc_zone_t* mi_get_default_zone(void) {
-  mi_atomic_do_once {
-    malloc_zone_register(&mi_malloc_zone);  // by calling register we avoid a zone error on free (see <http://eatmyrandom.blogspot.com/2010/03/mallocfree-interception-on-mac-os-x.html>)
-  }
-  return &mi_malloc_zone;
-}
-
-mi_decl_externc int  malloc_jumpstart(uintptr_t cookie);
-mi_decl_externc void _malloc_fork_prepare(void);
-mi_decl_externc void _malloc_fork_parent(void);
-mi_decl_externc void _malloc_fork_child(void);
-
-
-static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) {
-  MI_UNUSED(size); MI_UNUSED(flags);
-  return mi_get_default_zone();
-}
-
-static malloc_zone_t* mi_malloc_default_zone (void) {
-  return mi_get_default_zone();
-}
-
-static malloc_zone_t* mi_malloc_default_purgeable_zone(void) {
-  return mi_get_default_zone();
-}
-
-static void mi_malloc_destroy_zone(malloc_zone_t* zone) {
-  MI_UNUSED(zone);
-  // nothing.
-}
-
-static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) {
-  MI_UNUSED(task); MI_UNUSED(mr);
-  if (addresses != NULL) *addresses = NULL;
-  if (count != NULL) *count = 0;
-  return KERN_SUCCESS;
-}
-
-static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) {
-  return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name);
-}
-
-static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) {
-  MI_UNUSED(zone); MI_UNUSED(name);
-}
-
-static int mi_malloc_jumpstart(uintptr_t cookie) {
-  MI_UNUSED(cookie);
-  return 1; // or 0 for no error?
-}
-
-static void mi__malloc_fork_prepare(void) {
-  // nothing
-}
-static void mi__malloc_fork_parent(void) {
-  // nothing
-}
-static void mi__malloc_fork_child(void) {
-  // nothing
-}
-
-static void mi_malloc_printf(const char* fmt, ...) {
-  MI_UNUSED(fmt);
-}
-
-static bool zone_check(malloc_zone_t* zone) {
-  MI_UNUSED(zone);
-  return true;
-}
-
-static malloc_zone_t* zone_from_ptr(const void* p) {
-  MI_UNUSED(p);
-  return (mi_any_heap_contains(p) ? mi_get_default_zone() : NULL);
-}
-
-static void zone_log(malloc_zone_t* zone, void* p) {
-  MI_UNUSED(zone); MI_UNUSED(p);
-}
-
-static void zone_print(malloc_zone_t* zone, bool b) {
-  MI_UNUSED(zone); MI_UNUSED(b);
-}
-
-static void zone_print_ptr_info(void* p) {
-  MI_UNUSED(p);
-}
-
-static void zone_register(malloc_zone_t* zone) {
-  MI_UNUSED(zone);
-}
-
-static void zone_unregister(malloc_zone_t* zone) {
-  MI_UNUSED(zone);
-}
-
-// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`
-// See: <https://books.google.com/books?id=K8vUkpOXhN4C&pg=PA73>
-struct mi_interpose_s {
-  const void* replacement;
-  const void* target;
-};
-#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }
-#define MI_INTERPOSE_MI(fun)            MI_INTERPOSE_FUN(fun,mi_##fun)
-#define MI_INTERPOSE_ZONE(fun)          MI_INTERPOSE_FUN(malloc_##fun,fun)
-__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[]  __attribute__((section("__DATA, __interpose"))) =
-{
-
-  MI_INTERPOSE_MI(malloc_create_zone),
-  MI_INTERPOSE_MI(malloc_default_purgeable_zone),
-  MI_INTERPOSE_MI(malloc_default_zone),
-  MI_INTERPOSE_MI(malloc_destroy_zone),
-  MI_INTERPOSE_MI(malloc_get_all_zones),
-  MI_INTERPOSE_MI(malloc_get_zone_name),
-  MI_INTERPOSE_MI(malloc_jumpstart),
-  MI_INTERPOSE_MI(malloc_printf),
-  MI_INTERPOSE_MI(malloc_set_zone_name),
-  MI_INTERPOSE_MI(_malloc_fork_child),
-  MI_INTERPOSE_MI(_malloc_fork_parent),
-  MI_INTERPOSE_MI(_malloc_fork_prepare),
-
-  MI_INTERPOSE_ZONE(zone_batch_free),
-  MI_INTERPOSE_ZONE(zone_batch_malloc),
-  MI_INTERPOSE_ZONE(zone_calloc),
-  MI_INTERPOSE_ZONE(zone_check),
-  MI_INTERPOSE_ZONE(zone_free),
-  MI_INTERPOSE_ZONE(zone_from_ptr),
-  MI_INTERPOSE_ZONE(zone_log),
-  MI_INTERPOSE_ZONE(zone_malloc),
-  MI_INTERPOSE_ZONE(zone_memalign),
-  MI_INTERPOSE_ZONE(zone_print),
-  MI_INTERPOSE_ZONE(zone_print_ptr_info),
-  MI_INTERPOSE_ZONE(zone_realloc),
-  MI_INTERPOSE_ZONE(zone_register),
-  MI_INTERPOSE_ZONE(zone_unregister),
-  MI_INTERPOSE_ZONE(zone_valloc)
-};
-
-
-#else
-
-// ------------------------------------------------------
-// hook into the zone api's without interposing
-// This is the official way of adding an allocator but
-// it seems less robust than using interpose.
-// ------------------------------------------------------
-
-static inline malloc_zone_t* mi_get_default_zone(void)
-{
-  // The first returned zone is the real default
-  malloc_zone_t** zones = NULL;
-  unsigned count = 0;
-  kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count);
-  if (ret == KERN_SUCCESS && count > 0) {
-    return zones[0];
-  }
-  else {
-    // fallback
-    return malloc_default_zone();
-  }
-}
-
-#if defined(__clang__)
-__attribute__((constructor(101))) // highest priority
-#else
-__attribute__((constructor))      // priority level is not supported by gcc
-#endif
-__attribute__((used))
-static void _mi_macos_override_malloc(void) {
-  malloc_zone_t* purgeable_zone = NULL;
-
-  #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
-  // force the purgeable zone to exist to avoid strange bugs
-  if (malloc_default_purgeable_zone) {
-    purgeable_zone = malloc_default_purgeable_zone();
-  }
-  #endif
-
-  // Register our zone.
-  // thomcc: I think this is still needed to put us in the zone list.
-  malloc_zone_register(&mi_malloc_zone);
-  // Unregister the default zone, this makes our zone the new default
-  // as that was the last registered.
-  malloc_zone_t *default_zone = mi_get_default_zone();
-  // thomcc: Unsure if the next test is *always* false or just false in the
-  // cases I've tried. I'm also unsure if the code inside is needed. at all
-  if (default_zone != &mi_malloc_zone) {
-    malloc_zone_unregister(default_zone);
-
-    // Reregister the default zone so free and realloc in that zone keep working.
-    malloc_zone_register(default_zone);
-  }
-
-  // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs
-  // earlier than the default zone.
-  if (purgeable_zone != NULL) {
-    malloc_zone_unregister(purgeable_zone);
-    malloc_zone_register(purgeable_zone);
-  }
-
-}
-#endif  // MI_OSX_INTERPOSE
-
-#endif // MI_MALLOC_OVERRIDE
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/prim.c
deleted file mode 100644
index 8a2f4e8a..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/osx/prim.c
+++ /dev/null
@@ -1,9 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-// We use the unix/prim.c with the mmap API on macOSX
-#include "../unix/prim.c"
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/prim.c
deleted file mode 100644
index 5147bae8..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/prim.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-// Select the implementation of the primitives
-// depending on the OS.
-
-#if defined(_WIN32)
-#include "windows/prim.c"  // VirtualAlloc (Windows)
-
-#elif defined(__APPLE__)
-#include "osx/prim.c"      // macOSX (actually defers to mmap in unix/prim.c)
-
-#elif defined(__wasi__)
-#define MI_USE_SBRK
-#include "wasi/prim.c"     // memory-grow or sbrk (Wasm)
-
-#elif defined(__EMSCRIPTEN__)
-#include "emscripten/prim.c" // emmalloc_*, + pthread support
-
-#else
-#include "unix/prim.c"     // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.)
-
-#endif
-
-// Generic process initialization
-#ifndef MI_PRIM_HAS_PROCESS_ATTACH
-#if defined(__GNUC__) || defined(__clang__)
-  // gcc,clang: use the constructor/destructor attribute
-  // which for both seem to run before regular constructors/destructors
-  #if defined(__clang__)
-    #define mi_attr_constructor __attribute__((constructor(101)))
-    #define mi_attr_destructor  __attribute__((destructor(101)))
-  #else
-    #define mi_attr_constructor __attribute__((constructor))
-    #define mi_attr_destructor  __attribute__((destructor))
-  #endif
-  static void mi_attr_constructor mi_process_attach(void) {
-    _mi_auto_process_init();
-  }
-  static void mi_attr_destructor mi_process_detach(void) {
-    _mi_auto_process_done();
-  }
-#elif defined(__cplusplus)
-  // C++: use static initialization to detect process start/end
-  // This is not guaranteed to be first/last but the best we can generally do?
-  struct mi_init_done_t {
-    mi_init_done_t() {
-      _mi_auto_process_init();
-    }
-    ~mi_init_done_t() {
-      _mi_auto_process_done();
-    }
-  };
-  static mi_init_done_t mi_init_done;
- #else
-  #pragma message("define a way to call _mi_auto_process_init/done on your platform")
-#endif
-#endif
-
-// Generic allocator init/done callback
-#ifndef MI_PRIM_HAS_ALLOCATOR_INIT
-bool _mi_is_redirected(void) {
-  return false;
-}
-bool _mi_allocator_init(const char** message) {
-  if (message != NULL) { *message = NULL; }
-  return true;
-}
-void _mi_allocator_done(void) {
-  // nothing to do
-}
-#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/readme.md b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/readme.md
deleted file mode 100644
index 380dd3a7..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/readme.md
+++ /dev/null
@@ -1,9 +0,0 @@
-## Portability Primitives
-
-This is the portability layer where all primitives needed from the OS are defined.
-
-- `include/mimalloc/prim.h`: primitive portability API definition.
-- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform
-            (and on macOS, `osx/prim.c` defers to `unix/prim.c`).
-
-Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's.
\ No newline at end of file
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c
deleted file mode 100644
index 43efea36..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/unix/prim.c
+++ /dev/null
@@ -1,1001 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-// This file is included in `src/prim/prim.c`
-
-#ifndef _DEFAULT_SOURCE
-#define _DEFAULT_SOURCE   // ensure mmap flags and syscall are defined
-#endif
-
-#if defined(__sun)
-// illumos provides new mman.h api when any of these are defined
-// otherwise the old api based on caddr_t which predates the void pointers one.
-// stock solaris provides only the former, chose to atomically to discard those
-// flags only here rather than project wide tough.
-#undef _XOPEN_SOURCE
-#undef _POSIX_C_SOURCE
-#endif
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"
-
-#include <sys/mman.h>  // mmap
-#include <unistd.h>    // sysconf, sleep
-#include <fcntl.h>     // open, close, read, access
-#include <stdlib.h>    // getenv, arc4random_buf
-
-#if defined(__linux__)
-  #include <features.h>
-  #include <sys/prctl.h>    // THP disable, PR_SET_VMA
-  #include <sys/sysinfo.h>  // sysinfo
-  #if defined(__GLIBC__) && !defined(PR_SET_VMA)
-  #include <linux/prctl.h>
-  #endif
-  #if defined(__GLIBC__)
-  #include <linux/mman.h>   // linux mmap flags
-  #else
-  #include <sys/mman.h>
-  #endif
-#elif defined(__APPLE__)
-  #include <AvailabilityMacros.h>
-  #include <TargetConditionals.h>
-  #if !defined(TARGET_OS_OSX) || TARGET_OS_OSX   // see issue #879, used to be (!TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR)
-  #include <mach/vm_statistics.h>    // VM_MAKE_TAG, VM_FLAGS_SUPERPAGE_SIZE_2MB, etc.
-  #endif
-  #if !defined(MAC_OS_X_VERSION_10_7)
-  #define MAC_OS_X_VERSION_10_7   1070
-  #endif
-  #include <sys/sysctl.h>
-#elif defined(__FreeBSD__) || defined(__DragonFly__)
-  #include <sys/param.h>
-  #if __FreeBSD_version >= 1200000
-  #include <sys/cpuset.h>
-  #include <sys/domainset.h>
-  #endif
-  #include <sys/sysctl.h>
-#endif
-
-#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__FreeBSD__)
-  #define MI_HAS_SYSCALL_H
-  #include <sys/syscall.h>
-#endif
-
-#if !defined(MADV_DONTNEED) && defined(POSIX_MADV_DONTNEED)  // QNX
-#define MADV_DONTNEED  POSIX_MADV_DONTNEED
-#endif
-#if !defined(MADV_FREE) && defined(POSIX_MADV_FREE)  // QNX
-#define MADV_FREE  POSIX_MADV_FREE
-#endif
-
-#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // TODO: can we query the OS for this?
-
-//------------------------------------------------------------------------------------
-// Use syscalls for some primitives to allow for libraries that override open/read/close etc.
-// and do allocation themselves; using syscalls prevents recursion when mimalloc is
-// still initializing (issue #713)
-// Declare inline to avoid unused function warnings.
-//------------------------------------------------------------------------------------
-
-#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access)
-
-static inline int mi_prim_open(const char* fpath, int open_flags) {
-  return syscall(SYS_open,fpath,open_flags,0);
-}
-static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
-  return syscall(SYS_read,fd,buf,bufsize);
-}
-static inline int mi_prim_close(int fd) {
-  return syscall(SYS_close,fd);
-}
-static inline int mi_prim_access(const char *fpath, int mode) {
-  return syscall(SYS_access,fpath,mode);
-}
-
-#else
-
-static inline int mi_prim_open(const char* fpath, int open_flags) {
-  return open(fpath,open_flags);
-}
-static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
-  return read(fd,buf,bufsize);
-}
-static inline int mi_prim_close(int fd) {
-  return close(fd);
-}
-static inline int mi_prim_access(const char *fpath, int mode) {
-  return access(fpath,mode);
-}
-
-#endif
-
-
-
-//---------------------------------------------
-// init
-//---------------------------------------------
-
-static bool unix_detect_overcommit(void) {
-  bool os_overcommit = true;
-  #if defined(__linux__)
-    int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY);
-    if (fd >= 0) {
-      char buf[32];
-      ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));
-      mi_prim_close(fd);
-      // <https://www.kernel.org/doc/Documentation/vm/overcommit-accounting>
-      // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE)
-      if (nread >= 1) {
-        os_overcommit = (buf[0] == '0' || buf[0] == '1');
-      }
-    }
-  #elif defined(__FreeBSD__)
-    int val = 0;
-    size_t olen = sizeof(val);
-    if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) {
-      os_overcommit = (val != 0);
-    }
-  #else
-    // default: overcommit is true
-  #endif
-  return os_overcommit;
-}
-
-static bool unix_detect_thp(void) {
-  bool thp_enabled = false;
-  #if defined(__linux__)
-  int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
-  if (fd >= 0) {
-    char buf[32];
-    ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));
-    mi_prim_close(fd);
-    // <https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html>
-    // between brackets is the current value, for example: always [madvise] never
-    if (nread >= 1) {
-      thp_enabled = (_mi_strnstr(buf,32,"[never]") == NULL);
-    }
-  }
-  #endif
-  return thp_enabled;
-}
-
-// try to detect the physical memory dynamically (if possible)
-static void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) {
-  #if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE))  // freeBSD, macOS
-    MI_UNUSED(page_size);
-    int64_t physical_memory = 0;
-    size_t length = sizeof(int64_t);
-    #if defined(HW_PHYSMEM64)
-    int mib[2] = { CTL_HW, HW_PHYSMEM64 };
-    #else
-    int mib[2] = { CTL_HW, HW_MEMSIZE };
-    #endif
-    const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0);
-    if (err==0 && physical_memory > 0) {
-      const int64_t phys_in_kib = physical_memory / MI_KiB;
-      if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) {
-        *physical_memory_in_kib = (size_t)phys_in_kib;
-      }
-    }
-  #elif defined(__linux__)
-    MI_UNUSED(page_size);
-    struct sysinfo info; _mi_memzero_var(info);
-    const int err = sysinfo(&info);
-    if (err==0 && info.mem_unit > 0 && info.totalram <= SIZE_MAX) {
-      if (info.mem_unit==MI_KiB) {
-        *physical_memory_in_kib = (size_t)info.totalram;
-      }
-      else {
-        size_t total = 0;
-        if (!mi_mul_overflow((size_t)info.totalram, (size_t)info.mem_unit, &total)) {
-          *physical_memory_in_kib = (total / MI_KiB);
-        }
-      }
-    }
-  #elif defined(_SC_PHYS_PAGES)  // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100)
-    const long pphys = sysconf(_SC_PHYS_PAGES);
-    const size_t psize_in_kib = page_size / MI_KiB;
-    if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) {
-      *physical_memory_in_kib = (size_t)pphys * psize_in_kib;
-    }
-  #endif
-}
-
-void _mi_prim_mem_init( mi_os_mem_config_t* config )
-{
-  long psize = sysconf(_SC_PAGESIZE);
-  if (psize > 0 && (unsigned long)psize < SIZE_MAX) {
-    config->page_size = (size_t)psize;
-    config->alloc_granularity = (size_t)psize;
-    unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib);
-  }
-  config->large_page_size = MI_UNIX_LARGE_PAGE_SIZE;
-  config->has_overcommit = unix_detect_overcommit();
-  config->has_partial_free = true;    // mmap can free in parts
-  config->has_virtual_reserve = true; // todo: check if this true for NetBSD?  (for anonymous mmap with PROT_NONE)
-  config->has_transparent_huge_pages = unix_detect_thp();
-
-  // disable transparent huge pages for this process?
-  #if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE)
-  if (!mi_option_is_enabled(mi_option_allow_thp)) // disable THP if requested through an option
-  {
-    config->has_transparent_huge_pages = false;
-    if (prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0) == 0) {   // -1 on error, 1 if already disabled
-      // Most likely since distros often come with always/madvise settings.
-      // Disabling only for mimalloc process rather than touching system wide settings
-      (void)prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0);
-    }
-  }
-  #endif
-}
-
-
-//---------------------------------------------
-// free
-//---------------------------------------------
-
-int _mi_prim_free(void* addr, size_t size ) {
-  if (size==0) return 0;
-  bool err = (munmap(addr, size) == -1);
-  return (err ? errno : 0);
-}
-
-
-//---------------------------------------------
-// mmap
-//---------------------------------------------
-
-// return errno on failure
-static int unix_madvise(void* addr, size_t size, int advice) {
-  #if defined(__sun)
-  const int res = madvise((caddr_t)addr, size, advice);  // Solaris needs cast (issue #520)
-  return (res==0 ? 0 : errno);
-  #elif defined(__QNX__)
-  return posix_madvise(addr, size, advice);              // posix returns errno
-  #else
-  const int res = madvise(addr, size, advice);           // linux returns -1 on failure and sets errno
-  return (res==0 ? 0 : errno);
-  #endif
-}
-
-static void* unix_mmap_prim(void* addr, size_t size, int protect_flags, int flags, int fd) {
-  void* p = mmap(addr, size, protect_flags, flags, fd, 0 /* offset */);
-  #if defined(__linux__) && defined(PR_SET_VMA)
-  if (p!=MAP_FAILED && p!=NULL) {
-    prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, size, "mimalloc");
-  }
-  #endif
-  return p;
-}
-
-static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
-  MI_UNUSED(try_alignment);
-  void* p = NULL;
-  #if defined(MAP_ALIGNED)  // BSD
-  if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
-    size_t n = 0;
-    mi_bsr(try_alignment, &n);
-    if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) {  // alignment is a power of 2 and 4096 <= alignment <= 1GiB
-      p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd);
-      if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
-        int err = errno;
-        _mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
-      }
-      if (p!=MAP_FAILED) return p;
-      // fall back to regular mmap
-    }
-  }
-  #elif defined(MAP_ALIGN)  // Solaris
-  if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
-    p = unix_mmap_prim((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd);  // addr parameter is the required alignment
-    if (p!=MAP_FAILED) return p;
-    // fall back to regular mmap
-  }
-  #endif
-  #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
-  // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations
-  if (addr == NULL) {
-    void* hint = _mi_os_get_aligned_hint(try_alignment, size);
-    if (hint != NULL) {
-      p = unix_mmap_prim(hint, size, protect_flags, flags, fd);
-      if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
-        #if MI_TRACK_ENABLED  // asan sometimes does not instrument errno correctly?
-        int err = 0;
-        #else
-        int err = errno;
-        #endif
-        _mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
-      }
-      if (p!=MAP_FAILED) return p;
-      // fall back to regular mmap
-    }
-  }
-  #endif
-  // regular mmap
-  p = unix_mmap_prim(addr, size, protect_flags, flags, fd);
-  if (p!=MAP_FAILED) return p;
-  // failed to allocate
-  return NULL;
-}
-
-static int unix_mmap_fd(void) {
-  #if defined(VM_MAKE_TAG)
-  // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
-  int os_tag = (int)mi_option_get(mi_option_os_tag);
-  if (os_tag < 100 || os_tag > 255) { os_tag = 254; }
-  return VM_MAKE_TAG(os_tag);
-  #else
-  return -1;
-  #endif
-}
-
-static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
-  #if !defined(MAP_ANONYMOUS)
-  #define MAP_ANONYMOUS  MAP_ANON
-  #endif
-  #if !defined(MAP_NORESERVE)
-  #define MAP_NORESERVE  0
-  #endif
-  void* p = NULL;
-  const int fd = unix_mmap_fd();
-  int flags = MAP_PRIVATE | MAP_ANONYMOUS;
-  if (_mi_os_has_overcommit()) {
-    flags |= MAP_NORESERVE;
-  }
-  #if defined(PROT_MAX)
-  protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
-  #endif
-  // huge page allocation
-  if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) {
-    static _Atomic(size_t) large_page_try_ok; // = 0;
-    size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
-    if (!large_only && try_ok > 0) {
-      // If the OS is not configured for large OS pages, or the user does not have
-      // enough permission, the `mmap` will always fail (but it might also fail for other reasons).
-      // Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times
-      // to avoid too many failing calls to mmap.
-      mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);
-    }
-    else {
-      int lflags = flags & ~MAP_NORESERVE;  // using NORESERVE on huge pages seems to fail on Linux
-      int lfd = fd;
-      #ifdef MAP_ALIGNED_SUPER
-      lflags |= MAP_ALIGNED_SUPER;
-      #endif
-      #ifdef MAP_HUGETLB
-      lflags |= MAP_HUGETLB;
-      #endif
-      #ifdef MAP_HUGE_1GB
-      static bool mi_huge_pages_available = true;
-      if (large_only && (size % MI_GiB) == 0 && mi_huge_pages_available) {
-        lflags |= MAP_HUGE_1GB;
-      }
-      else
-      #endif
-      {
-        #ifdef MAP_HUGE_2MB
-        lflags |= MAP_HUGE_2MB;
-        #endif
-      }
-      #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB
-      lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
-      #endif
-      if (large_only || lflags != flags) {
-        // try large OS page allocation
-        *is_large = true;
-        p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
-        #ifdef MAP_HUGE_1GB
-        if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {
-          mi_huge_pages_available = false; // don't try huge 1GiB pages again
-          if (large_only) {
-            _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno);
-          }
-          lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
-          p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
-        }
-        #endif
-        if (large_only) return p;
-        if (p == NULL) {
-          mi_atomic_store_release(&large_page_try_ok, (size_t)8);  // on error, don't try again for the next N allocations
-        }
-      }
-    }
-  }
-  // regular allocation
-  if (p == NULL) {
-    *is_large = false;
-    p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd);
-    #if !defined(MI_NO_THP)
-    if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) {
-      #if defined(MADV_HUGEPAGE)
-      // Many Linux systems don't allow MAP_HUGETLB but they support instead
-      // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE
-      // though since properly aligned allocations will already use large pages if available
-      // in that case -- in particular for our large regions (in `memory.c`).
-      // However, some systems only allow THP if called with explicit `madvise`, so
-      // when large OS pages are enabled for mimalloc, we call `madvise` anyways.
-      if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {
-        // *is_large = true; // possibly
-      };
-      #elif defined(__sun)
-      struct memcntl_mha cmd = {0};
-      cmd.mha_pagesize = _mi_os_large_page_size();
-      cmd.mha_cmd = MHA_MAPSIZE_VA;
-      if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
-        // *is_large = true; // possibly
-      }
-      #endif
-    }
-    #endif
-  }
-  return p;
-}
-
-// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
-int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
-  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
-  mi_assert_internal(commit || !allow_large);
-  mi_assert_internal(try_alignment > 0);
-  *is_zero = true;
-  int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
-  *addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large);
-  return (*addr != NULL ? 0 : errno);
-}
-
-
-//---------------------------------------------
-// Commit/Reset
-//---------------------------------------------
-
-static void unix_mprotect_hint(int err) {
-  #if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page
-  if (err == ENOMEM) {
-    _mi_warning_message("The next warning may be caused by a low memory map limit.\n"
-                        "  On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n"
-                        "  For example: sudo sysctl -w vm.max_map_count=262144\n");
-  }
-  #else
-  MI_UNUSED(err);
-  #endif
-}
-
-
-
-
-
-int _mi_prim_commit(void* start, size_t size, bool* is_zero) {
-  // commit: ensure we can access the area
-  // note: we may think that *is_zero can be true since the memory
-  // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but
-  // we sometimes call commit on a range with still partially committed
-  // memory and `mprotect` does not zero the range.
-  *is_zero = false;
-  int err = mprotect(start, size, (PROT_READ | PROT_WRITE));
-  if (err != 0) {
-    err = errno;
-    unix_mprotect_hint(err);
-  }
-  return err;
-}
-
-int _mi_prim_reuse(void* start, size_t size) {
-  MI_UNUSED(start); MI_UNUSED(size);
-  #if defined(__APPLE__) && defined(MADV_FREE_REUSE)
-  return unix_madvise(start, size, MADV_FREE_REUSE);
-  #endif
-  return 0;
-}
-
-int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
-  int err = 0;
-  #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
-    // decommit on macOS: map fresh zero pages over the range (WeavePy patch).
-    // MADV_FREE_REUSABLE only lowers the footprint: the pages stay in the
-    // resident size (and `ru_maxrss`) until the system reclaims them. If the
-    // mapping fails, the range is left as it was and marked reusable.
-    void* p = mmap(start, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, unix_mmap_fd(), 0);
-    if (p != start) {
-      err = unix_madvise(start, size, MADV_FREE_REUSABLE);
-      if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
-    }
-  #else
-    // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
-    err = unix_madvise(start, size, MADV_DONTNEED);
-  #endif
-  #if !MI_DEBUG && MI_SECURE<=2
-    *needs_recommit = false;
-  #else
-    *needs_recommit = true;
-    mprotect(start, size, PROT_NONE);
-  #endif
-  /*
-  // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)
-  *needs_recommit = true;
-  const int fd = unix_mmap_fd();
-  void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
-  if (p != start) { err = errno; }
-  */
-  return err;
-}
-
-int _mi_prim_reset(void* start, size_t size) {
-  int err = 0;
-
-  // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower)
-  #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
-  err = unix_madvise(start, size, MADV_FREE_REUSABLE);
-  if (err==0) return 0;
-  // fall through
-  #endif
-
-  #if defined(MADV_FREE)
-  // Otherwise, we try to use `MADV_FREE` as that is the fastest. A drawback though is that it
-  // will not reduce the `rss` stats in tools like `top` even though the memory is available
-  // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by
-  // default `MADV_DONTNEED` is used though.
-  static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE);
-  int oadvice = (int)mi_atomic_load_relaxed(&advice);
-  while ((err = unix_madvise(start, size, oadvice)) != 0 && err == EAGAIN) { /* try again */ };
-  if (err == EINVAL && oadvice == MADV_FREE) {
-    // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
-    mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED);
-    err = unix_madvise(start, size, MADV_DONTNEED);
-  }
-  #else
-  err = unix_madvise(start, size, MADV_DONTNEED);
-  #endif
-  return err;
-}
-
-int _mi_prim_protect(void* start, size_t size, bool protect) {
-  int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));
-  if (err != 0) { err = errno; }
-  unix_mprotect_hint(err);
-  return err;
-}
-
-
-
-//---------------------------------------------
-// Huge page allocation
-//---------------------------------------------
-
-#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)
-
-#ifndef MPOL_PREFERRED
-#define MPOL_PREFERRED 1
-#endif
-
-#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind)
-static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
-  return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags);
-}
-#else
-static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
-  MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags);
-  return 0;
-}
-#endif
-
-int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
-  bool is_large = true;
-  *is_zero = true;
-  *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large);
-  if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes
-    unsigned long numa_mask = (1UL << numa_node);
-    // TODO: does `mbind` work correctly for huge OS pages? should we
-    // use `set_mempolicy` before calling mmap instead?
-    // see: <https://lkml.org/lkml/2017/2/9/875>
-    long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);
-    if (err != 0) {
-      err = errno;
-      _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err);
-    }
-  }
-  return (*addr != NULL ? 0 : errno);
-}
-
-#else
-
-int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
-  MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
-  *is_zero = false;
-  *addr = NULL;
-  return ENOMEM;
-}
-
-#endif
-
-//---------------------------------------------
-// NUMA nodes
-//---------------------------------------------
-
-#if defined(__linux__)
-
-size_t _mi_prim_numa_node(void) {
-  #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)
-    unsigned long node = 0;
-    unsigned long ncpu = 0;
-    long err = syscall(SYS_getcpu, &ncpu, &node, NULL);
-    if (err != 0) return 0;
-    return node;
-  #else
-    return 0;
-  #endif
-}
-
-size_t _mi_prim_numa_node_count(void) {
-  char buf[128];
-  unsigned node = 0;
-  for(node = 0; node < 256; node++) {
-    // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
-    _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
-    if (mi_prim_access(buf,R_OK) != 0) break;
-  }
-  return (node+1);
-}
-
-#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000
-
-size_t _mi_prim_numa_node(void) {
-  domainset_t dom;
-  size_t node;
-  int policy;
-  if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul;
-  for (node = 0; node < MAXMEMDOM; node++) {
-    if (DOMAINSET_ISSET(node, &dom)) return node;
-  }
-  return 0ul;
-}
-
-size_t _mi_prim_numa_node_count(void) {
-  size_t ndomains = 0;
-  size_t len = sizeof(ndomains);
-  if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul;
-  return ndomains;
-}
-
-#elif defined(__DragonFly__)
-
-size_t _mi_prim_numa_node(void) {
-  // TODO: DragonFly does not seem to provide any userland means to get this information.
-  return 0ul;
-}
-
-size_t _mi_prim_numa_node_count(void) {
-  size_t ncpus = 0, nvirtcoresperphys = 0;
-  size_t len = sizeof(size_t);
-  if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul;
-  if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul;
-  return nvirtcoresperphys * ncpus;
-}
-
-#else
-
-size_t _mi_prim_numa_node(void) {
-  return 0;
-}
-
-size_t _mi_prim_numa_node_count(void) {
-  return 1;
-}
-
-#endif
-
-// ----------------------------------------------------------------
-// Clock
-// ----------------------------------------------------------------
-
-#include <time.h>
-
-#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
-
-mi_msecs_t _mi_prim_clock_now(void) {
-  struct timespec t;
-  #ifdef CLOCK_MONOTONIC
-  clock_gettime(CLOCK_MONOTONIC, &t);
-  #else
-  clock_gettime(CLOCK_REALTIME, &t);
-  #endif
-  return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);
-}
-
-#else
-
-// low resolution timer
-mi_msecs_t _mi_prim_clock_now(void) {
-  #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)
-  return (mi_msecs_t)clock();
-  #elif (CLOCKS_PER_SEC < 1000)
-  return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
-  #else
-  return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
-  #endif
-}
-
-#endif
-
-
-
-
-//----------------------------------------------------------------
-// Process info
-//----------------------------------------------------------------
-
-#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)
-#include <stdio.h>
-#include <unistd.h>
-#include <sys/resource.h>
-
-#if defined(__APPLE__)
-#include <mach/mach.h>
-#endif
-
-#if defined(__HAIKU__)
-#include <kernel/OS.h>
-#endif
-
-static mi_msecs_t timeval_secs(const struct timeval* tv) {
-  return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L);
-}
-
-void _mi_prim_process_info(mi_process_info_t* pinfo)
-{
-  struct rusage rusage;
-  getrusage(RUSAGE_SELF, &rusage);
-  pinfo->utime = timeval_secs(&rusage.ru_utime);
-  pinfo->stime = timeval_secs(&rusage.ru_stime);
-#if !defined(__HAIKU__)
-  pinfo->page_faults = rusage.ru_majflt;
-#endif
-#if defined(__HAIKU__)
-  // Haiku does not have (yet?) a way to
-  // get these stats per process
-  thread_info tid;
-  area_info mem;
-  ssize_t c;
-  get_thread_info(find_thread(0), &tid);
-  while (get_next_area_info(tid.team, &c, &mem) == B_OK) {
-    pinfo->peak_rss += mem.ram_size;
-  }
-  pinfo->page_faults = 0;
-#elif defined(__APPLE__)
-  pinfo->peak_rss = rusage.ru_maxrss;         // macos reports in bytes
-  #ifdef MACH_TASK_BASIC_INFO
-  struct mach_task_basic_info info;
-  mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
-  if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
-    pinfo->current_rss = (size_t)info.resident_size;
-  }
-  #else
-  struct task_basic_info info;
-  mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;
-  if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
-    pinfo->current_rss = (size_t)info.resident_size;
-  }
-  #endif
-#else
-  pinfo->peak_rss = rusage.ru_maxrss * 1024;  // Linux/BSD report in KiB
-#endif
-  // use defaults for commit
-}
-
-#else
-
-#ifndef __wasi__
-// WebAssembly instances are not processes
-#pragma message("define a way to get process info")
-#endif
-
-void _mi_prim_process_info(mi_process_info_t* pinfo)
-{
-  // use defaults
-  MI_UNUSED(pinfo);
-}
-
-#endif
-
-
-//----------------------------------------------------------------
-// Output
-//----------------------------------------------------------------
-
-void _mi_prim_out_stderr( const char* msg ) {
-  fputs(msg,stderr);
-}
-
-
-//----------------------------------------------------------------
-// Environment
-//----------------------------------------------------------------
-
-#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0)
-// On Posix systemsr use `environ` to access environment variables
-// even before the C runtime is initialized.
-#if defined(__APPLE__) && defined(__has_include) && __has_include(<crt_externs.h>)
-#include <crt_externs.h>
-static char** mi_get_environ(void) {
-  return (*_NSGetEnviron());
-}
-#else
-extern char** environ;
-static char** mi_get_environ(void) {
-  return environ;
-}
-#endif
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
-  if (name==NULL) return false;
-  const size_t len = _mi_strlen(name);
-  if (len == 0) return false;
-  char** env = mi_get_environ();
-  if (env == NULL) return false;
-  // compare up to 10000 entries
-  for (int i = 0; i < 10000 && env[i] != NULL; i++) {
-    const char* s = env[i];
-    if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive
-      // found it
-      _mi_strlcpy(result, s + len + 1, result_size);
-      return true;
-    }
-  }
-  return false;
-}
-#else
-// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
-  // cannot call getenv() when still initializing the C runtime.
-  if (_mi_preloading()) return false;
-  const char* s = getenv(name);
-  if (s == NULL) {
-    // we check the upper case name too.
-    char buf[64+1];
-    size_t len = _mi_strnlen(name,sizeof(buf)-1);
-    for (size_t i = 0; i < len; i++) {
-      buf[i] = _mi_toupper(name[i]);
-    }
-    buf[len] = 0;
-    s = getenv(buf);
-  }
-  if (s == NULL || _mi_strnlen(s,result_size) >= result_size)  return false;
-  _mi_strlcpy(result, s, result_size);
-  return true;
-}
-#endif  // !MI_USE_ENVIRON
-
-
-//----------------------------------------------------------------
-// Random
-//----------------------------------------------------------------
-
-#if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15)
-#include <CommonCrypto/CommonCryptoError.h>
-#include <CommonCrypto/CommonRandom.h>
-
-bool _mi_prim_random_buf(void* buf, size_t buf_len) {
-  // We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf
-  // may fail silently on macOS. See PR #390, and <https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html>
-  return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess);
-}
-
-#elif defined(__ANDROID__) || defined(__DragonFly__) || \
-      defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
-      defined(__sun) || \
-      (defined(__APPLE__) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_7))
-
-bool _mi_prim_random_buf(void* buf, size_t buf_len) {
-  arc4random_buf(buf, buf_len);
-  return true;
-}
-
-#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__)   // also for old apple versions < 10.7 (issue #829)
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <errno.h>
-
-bool _mi_prim_random_buf(void* buf, size_t buf_len) {
-  // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h`
-  // and for the latter the actual `getrandom` call is not always defined.
-  // (see <https://stackoverflow.com/questions/45237324/why-doesnt-getrandom-compile>)
-  // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed.
-  #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom)
-    #ifndef GRND_NONBLOCK
-    #define GRND_NONBLOCK (1)
-    #endif
-    static _Atomic(uintptr_t) no_getrandom; // = 0
-    if (mi_atomic_load_acquire(&no_getrandom)==0) {
-      ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK);
-      if (ret >= 0) return (buf_len == (size_t)ret);
-      if (errno != ENOSYS) return false;
-      mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom
-    }
-  #endif
-  int flags = O_RDONLY;
-  #if defined(O_CLOEXEC)
-  flags |= O_CLOEXEC;
-  #endif
-  int fd = mi_prim_open("/dev/urandom", flags);
-  if (fd < 0) return false;
-  size_t count = 0;
-  while(count < buf_len) {
-    ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count);
-    if (ret<=0) {
-      if (errno!=EAGAIN && errno!=EINTR) break;
-    }
-    else {
-      count += ret;
-    }
-  }
-  mi_prim_close(fd);
-  return (count==buf_len);
-}
-
-#else
-
-bool _mi_prim_random_buf(void* buf, size_t buf_len) {
-  return false;
-}
-
-#endif
-
-
-//----------------------------------------------------------------
-// Thread init/done
-//----------------------------------------------------------------
-
-#if defined(MI_USE_PTHREADS)
-
-// use pthread local storage keys to detect thread ending
-// (and used with MI_TLS_PTHREADS for the default theap)
-pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
-
-static void mi_pthread_done(void* value) {
-  if (value!=NULL) {
-    _mi_thread_done((mi_theap_t*)value);
-  }
-}
-
-void _mi_prim_thread_init_auto_done(void) {
-  mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));
-  pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);
-}
-
-void _mi_prim_thread_done_auto_done(void) {
-  if (_mi_heap_default_key != (pthread_key_t)(-1)) {  // do not leak the key, see issue #809
-    pthread_key_delete(_mi_heap_default_key);
-  }
-}
-
-void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-  if (_mi_heap_default_key != (pthread_key_t)(-1)) {  // can happen during recursive invocation on freeBSD
-    pthread_setspecific(_mi_heap_default_key, theap);
-  }
-}
-
-#else
-
-void _mi_prim_thread_init_auto_done(void) {
-  // nothing
-}
-
-void _mi_prim_thread_done_auto_done(void) {
-  // nothing
-}
-
-void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-  MI_UNUSED(theap);
-}
-
-#endif
-
-bool _mi_prim_thread_is_in_threadpool(void) {
-  return false;
-}
-
-void _mi_prim_thread_yield(void) {
-  sleep(0);
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/wasi/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/wasi/prim.c
deleted file mode 100644
index 790daff0..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/wasi/prim.c
+++ /dev/null
@@ -1,291 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-// This file is included in `src/prim/prim.c`
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"
-
-#include <stdio.h>   // fputs
-#include <stdlib.h>  // getenv
-#include <unistd.h>  // sbrk, sleep
-
-//---------------------------------------------
-// Initialize
-//---------------------------------------------
-
-void _mi_prim_mem_init( mi_os_mem_config_t* config ) {
-  config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
-  config->alloc_granularity = 16;
-  config->has_overcommit = false;
-  config->has_partial_free = false;
-  config->has_virtual_reserve = false;
-}
-
-//---------------------------------------------
-// Free
-//---------------------------------------------
-
-int _mi_prim_free(void* addr, size_t size ) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  // wasi theap cannot be shrunk
-  return 0;
-}
-
-
-//---------------------------------------------
-// Allocation: sbrk or memory_grow
-//---------------------------------------------
-
-#if defined(MI_USE_SBRK)
-  static void* mi_memory_grow( size_t size ) {
-    void* p = sbrk(size);
-    if (p == (void*)(-1)) return NULL;
-    #if !defined(__wasi__) // on wasi this is always zero initialized already (?)
-    memset(p,0,size);
-    #endif
-    return p;
-  }
-#elif defined(__wasi__)
-  static void* mi_memory_grow( size_t size ) {
-    size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size()))
-                            : __builtin_wasm_memory_size(0));
-    if (base == SIZE_MAX) return NULL;
-    return (void*)(base * _mi_os_page_size());
-  }
-#endif
-
-#if defined(MI_USE_PTHREADS)
-static pthread_mutex_t mi_theap_grow_mutex = PTHREAD_MUTEX_INITIALIZER;
-#endif
-
-static void* mi_prim_mem_grow(size_t size, size_t try_alignment) {
-  void* p = NULL;
-  if (try_alignment <= 1) {
-    // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now)
-    #if defined(MI_USE_PTHREADS)
-    pthread_mutex_lock(&mi_theap_grow_mutex);
-    #endif
-    p = mi_memory_grow(size);
-    #if defined(MI_USE_PTHREADS)
-    pthread_mutex_unlock(&mi_theap_grow_mutex);
-    #endif
-  }
-  else {
-    void* base = NULL;
-    size_t alloc_size = 0;
-    // to allocate aligned use a lock to try to avoid thread interaction
-    // between getting the current size and actual allocation
-    // (also, `sbrk` is not thread safe in general)
-    #if defined(MI_USE_PTHREADS)
-    pthread_mutex_lock(&mi_theap_grow_mutex);
-    #endif
-    {
-      void* current = mi_memory_grow(0);  // get current size
-      if (current != NULL) {
-        void* aligned_current = _mi_align_up_ptr(current, try_alignment);  // and align from there to minimize wasted space
-        alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size());
-        base = mi_memory_grow(alloc_size);
-      }
-    }
-    #if defined(MI_USE_PTHREADS)
-    pthread_mutex_unlock(&mi_theap_grow_mutex);
-    #endif
-    if (base != NULL) {
-      p = _mi_align_up_ptr(base, try_alignment);
-      if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) {
-        // another thread used wasm_memory_grow/sbrk in-between and we do not have enough
-        // space after alignment. Give up (and waste the space as we cannot shrink :-( )
-        // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align)
-        p = NULL;
-      }
-    }
-  }
-  /*
-  if (p == NULL) {
-    _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment);
-    errno = ENOMEM;
-    return NULL;
-  }
-  */
-  mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 );
-  return p;
-}
-
-// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
-int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
-  MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);
-  *is_large = false;
-  *is_zero = false;
-  *addr = mi_prim_mem_grow(size, try_alignment);
-  return (*addr != NULL ? 0 : ENOMEM);
-}
-
-
-//---------------------------------------------
-// Commit/Reset/Protect
-//---------------------------------------------
-
-int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  *is_zero = false;
-  return 0;
-}
-
-int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  *needs_recommit = false;
-  return 0;
-}
-
-int _mi_prim_reset(void* addr, size_t size) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  return 0;
-}
-
-int _mi_prim_reuse(void* addr, size_t size) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  return 0;
-}
-
-int _mi_prim_protect(void* addr, size_t size, bool protect) {
-  MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);
-  return 0;
-}
-
-
-//---------------------------------------------
-// Huge pages and NUMA nodes
-//---------------------------------------------
-
-int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
-  MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
-  *is_zero = true;
-  *addr = NULL;
-  return ENOSYS;
-}
-
-size_t _mi_prim_numa_node(void) {
-  return 0;
-}
-
-size_t _mi_prim_numa_node_count(void) {
-  return 1;
-}
-
-
-//----------------------------------------------------------------
-// Clock
-//----------------------------------------------------------------
-
-#include <time.h>
-
-#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
-
-mi_msecs_t _mi_prim_clock_now(void) {
-  struct timespec t;
-  #ifdef CLOCK_MONOTONIC
-  clock_gettime(CLOCK_MONOTONIC, &t);
-  #else
-  clock_gettime(CLOCK_REALTIME, &t);
-  #endif
-  return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);
-}
-
-#else
-
-// low resolution timer
-mi_msecs_t _mi_prim_clock_now(void) {
-  #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)
-  return (mi_msecs_t)clock();
-  #elif (CLOCKS_PER_SEC < 1000)
-  return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
-  #else
-  return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
-  #endif
-}
-
-#endif
-
-
-//----------------------------------------------------------------
-// Process info
-//----------------------------------------------------------------
-
-void _mi_prim_process_info(mi_process_info_t* pinfo)
-{
-  // use defaults
-  MI_UNUSED(pinfo);
-}
-
-
-//----------------------------------------------------------------
-// Output
-//----------------------------------------------------------------
-
-void _mi_prim_out_stderr( const char* msg ) {
-  fputs(msg,stderr);
-}
-
-
-//----------------------------------------------------------------
-// Environment
-//----------------------------------------------------------------
-
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
-  // cannot call getenv() when still initializing the C runtime.
-  if (_mi_preloading()) return false;
-  const char* s = getenv(name);
-  if (s == NULL) {
-    // we check the upper case name too.
-    char buf[64+1];
-    size_t len = _mi_strnlen(name,sizeof(buf)-1);
-    for (size_t i = 0; i < len; i++) {
-      buf[i] = _mi_toupper(name[i]);
-    }
-    buf[len] = 0;
-    s = getenv(buf);
-  }
-  if (s == NULL || _mi_strnlen(s,result_size) >= result_size)  return false;
-  _mi_strlcpy(result, s, result_size);
-  return true;
-}
-
-
-//----------------------------------------------------------------
-// Random
-//----------------------------------------------------------------
-
-bool _mi_prim_random_buf(void* buf, size_t buf_len) {
-  return false;
-}
-
-
-//----------------------------------------------------------------
-// Thread init/done
-//----------------------------------------------------------------
-
-void _mi_prim_thread_init_auto_done(void) {
-  // nothing
-}
-
-void _mi_prim_thread_done_auto_done(void) {
-  // nothing
-}
-
-void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-  MI_UNUSED(theap);
-}
-
-bool _mi_prim_thread_is_in_threadpool(void) {
-  return false;
-}
-
-void _mi_prim_thread_yield(void) {
-  sleep(0);
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw-mimalloc.wprp b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw-mimalloc.wprp
deleted file mode 100644
index b00cd7ad..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw-mimalloc.wprp
+++ /dev/null
@@ -1,61 +0,0 @@
-<WindowsPerformanceRecorder Version="1.0">
-  <Profiles>
-    <SystemCollector Id="WPR_initiated_WprApp_WPR_System_Collector" Name="WPR_initiated_WprApp_WPR System Collector">
-      <BufferSize Value="1024" />
-      <Buffers Value="100" />
-    </SystemCollector>
-    <EventCollector Id="Mimalloc_Collector" Name="Mimalloc Collector">
-      <BufferSize Value="1024" />
-      <Buffers Value="100" />
-    </EventCollector>
-    <SystemProvider Id="WPR_initiated_WprApp_WPR_System_Collector_Provider">
-      <Keywords>
-        <Keyword Value="Loader" />
-      </Keywords>
-    </SystemProvider>
-    <EventProvider Id="MimallocEventProvider" Name="138f4dbb-ee04-4899-aa0a-572ad4475779" NonPagedMemory="true" Stack="true">
-      <EventFilters FilterIn="true">
-        <EventId Value="100" />
-        <EventId Value="101" />
-      </EventFilters>
-    </EventProvider>
-    <Profile Id="CustomHeap.Verbose.File" Name="CustomHeap" Description="RunningProfile:CustomHeap.Verbose.File" LoggingMode="File" DetailLevel="Verbose">
-      <ProblemCategories>
-        <ProblemCategory Value="Resource Analysis" />
-      </ProblemCategories>
-      <Collectors>
-        <SystemCollectorId Value="WPR_initiated_WprApp_WPR_System_Collector">
-          <SystemProviderId Value="WPR_initiated_WprApp_WPR_System_Collector_Provider" />
-        </SystemCollectorId>
-        <EventCollectorId Value="Mimalloc_Collector">
-          <EventProviders>
-            <EventProviderId Value="MimallocEventProvider" >
-              <Keywords>
-                <Keyword Value="100"/>
-                <Keyword Value="101"/>
-              </Keywords>
-            </EventProviderId>
-          </EventProviders>
-        </EventCollectorId>
-      </Collectors>
-      <TraceMergeProperties>
-        <TraceMergeProperty Id="BaseVerboseTraceMergeProperties" Name="BaseTraceMergeProperties">
-          <DeletePreMergedTraceFiles Value="true" />
-          <FileCompression Value="false" />
-          <InjectOnly Value="false" />
-          <SkipMerge Value="false" />
-          <CustomEvents>
-            <CustomEvent Value="ImageId" />
-            <CustomEvent Value="BuildInfo" />
-            <CustomEvent Value="VolumeMapping" />
-            <CustomEvent Value="EventMetadata" />
-            <CustomEvent Value="PerfTrackMetadata" />
-            <CustomEvent Value="WinSAT" />
-            <CustomEvent Value="NetworkInterface" />
-          </CustomEvents>
-        </TraceMergeProperty>
-      </TraceMergeProperties>
-    </Profile>
-  </Profiles>
-</WindowsPerformanceRecorder>
-
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.h b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.h
deleted file mode 100644
index 4e0a092a..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.h
+++ /dev/null
@@ -1,905 +0,0 @@
-//**********************************************************************`
-//* This is an include file generated by Message Compiler.             *`
-//*                                                                    *`
-//* Copyright (c) Microsoft Corporation. All Rights Reserved.          *`
-//**********************************************************************`
-#pragma once
-
-//*****************************************************************************
-//
-// Notes on the ETW event code generated by MC:
-//
-// - Structures and arrays of structures are treated as an opaque binary blob.
-//   The caller is responsible for packing the data for the structure into a
-//   single region of memory, with no padding between values. The macro will
-//   have an extra parameter for the length of the blob.
-// - Arrays of nul-terminated strings must be packed by the caller into a
-//   single binary blob containing the correct number of strings, with a nul
-//   after each string. The size of the blob is specified in characters, and
-//   includes the final nul.
-// - Arrays of SID are treated as a single binary blob. The caller is
-//   responsible for packing the SID values into a single region of memory with
-//   no padding.
-// - The length attribute on the data element in the manifest is significant
-//   for values with intype win:UnicodeString, win:AnsiString, or win:Binary.
-//   The length attribute must be specified for win:Binary, and is optional for
-//   win:UnicodeString and win:AnsiString (if no length is given, the strings
-//   are assumed to be nul-terminated). For win:UnicodeString, the length is
-//   measured in characters, not bytes.
-// - For an array of win:UnicodeString, win:AnsiString, or win:Binary, the
-//   length attribute applies to every value in the array, so every value in
-//   the array must have the same length. The values in the array are provided
-//   to the macro via a single pointer -- the caller is responsible for packing
-//   all of the values into a single region of memory with no padding between
-//   values.
-// - Values of type win:CountedUnicodeString, win:CountedAnsiString, and
-//   win:CountedBinary can be generated and collected on Vista or later.
-//   However, they may not decode properly without the Windows 10 2018 Fall
-//   Update.
-// - Arrays of type win:CountedUnicodeString, win:CountedAnsiString, and
-//   win:CountedBinary must be packed by the caller into a single region of
-//   memory. The format for each item is a UINT16 byte-count followed by that
-//   many bytes of data. When providing the array to the generated macro, you
-//   must provide the total size of the packed array data, including the UINT16
-//   sizes for each item. In the case of win:CountedUnicodeString, the data
-//   size is specified in WCHAR (16-bit) units. In the case of
-//   win:CountedAnsiString and win:CountedBinary, the data size is specified in
-//   bytes.
-//
-//*****************************************************************************
-
-#include <wmistr.h>
-#include <evntrace.h>
-#include <evntprov.h>
-
-#ifndef ETW_INLINE
-  #ifdef _ETW_KM_
-    // In kernel mode, save stack space by never inlining templates.
-    #define ETW_INLINE DECLSPEC_NOINLINE __inline
-  #else
-    // In user mode, save code size by inlining templates as appropriate.
-    #define ETW_INLINE __inline
-  #endif
-#endif // ETW_INLINE
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
-//
-// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:
-// Define this macro to have the compiler skip the generated functions in this
-// header.
-//
-#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION
-
-//
-// MCGEN_USE_KERNEL_MODE_APIS macro:
-// Controls whether the generated code uses kernel-mode or user-mode APIs.
-// - Set to 0 to use Windows user-mode APIs such as EventRegister.
-// - Set to 1 to use Windows kernel-mode APIs such as EtwRegister.
-// Default is based on whether the _ETW_KM_ macro is defined (i.e. by wdm.h).
-// Note that the APIs can also be overridden directly, e.g. by setting the
-// MCGEN_EVENTWRITETRANSFER or MCGEN_EVENTREGISTER macros.
-//
-#ifndef MCGEN_USE_KERNEL_MODE_APIS
-  #ifdef _ETW_KM_
-    #define MCGEN_USE_KERNEL_MODE_APIS 1
-  #else
-    #define MCGEN_USE_KERNEL_MODE_APIS 0
-  #endif
-#endif // MCGEN_USE_KERNEL_MODE_APIS
-
-//
-// MCGEN_HAVE_EVENTSETINFORMATION macro:
-// Controls how McGenEventSetInformation uses the EventSetInformation API.
-// - Set to 0 to disable the use of EventSetInformation
-//   (McGenEventSetInformation will always return an error).
-// - Set to 1 to directly invoke MCGEN_EVENTSETINFORMATION.
-// - Set to 2 to to locate EventSetInformation at runtime via GetProcAddress
-//   (user-mode) or MmGetSystemRoutineAddress (kernel-mode).
-// Default is determined as follows:
-// - If MCGEN_EVENTSETINFORMATION has been customized, set to 1
-//   (i.e. use MCGEN_EVENTSETINFORMATION).
-// - Else if the target OS version has EventSetInformation, set to 1
-//   (i.e. use MCGEN_EVENTSETINFORMATION).
-// - Else set to 2 (i.e. try to dynamically locate EventSetInformation).
-// Note that an McGenEventSetInformation function will only be generated if one
-// or more provider in a manifest has provider traits.
-//
-#ifndef MCGEN_HAVE_EVENTSETINFORMATION
-  #ifdef MCGEN_EVENTSETINFORMATION             // if MCGEN_EVENTSETINFORMATION has been customized,
-    #define MCGEN_HAVE_EVENTSETINFORMATION   1 //   directly invoke MCGEN_EVENTSETINFORMATION(...).
-  #elif MCGEN_USE_KERNEL_MODE_APIS             // else if using kernel-mode APIs,
-    #if NTDDI_VERSION >= 0x06040000            //   if target OS is Windows 10 or later,
-      #define MCGEN_HAVE_EVENTSETINFORMATION 1 //     directly invoke MCGEN_EVENTSETINFORMATION(...).
-    #else                                      //   else
-      #define MCGEN_HAVE_EVENTSETINFORMATION 2 //     find "EtwSetInformation" via MmGetSystemRoutineAddress.
-    #endif                                     // else (using user-mode APIs)
-  #else                                        //   if target OS and SDK is Windows 8 or later,
-    #if WINVER >= 0x0602 && defined(EVENT_FILTER_TYPE_SCHEMATIZED)
-      #define MCGEN_HAVE_EVENTSETINFORMATION 1 //     directly invoke MCGEN_EVENTSETINFORMATION(...).
-    #else                                      //   else
-      #define MCGEN_HAVE_EVENTSETINFORMATION 2 //     find "EventSetInformation" via GetModuleHandleExW/GetProcAddress.
-    #endif
-  #endif
-#endif // MCGEN_HAVE_EVENTSETINFORMATION
-
-//
-// MCGEN Override Macros
-//
-// The following override macros may be defined before including this header
-// to control the APIs used by this header:
-//
-// - MCGEN_EVENTREGISTER
-// - MCGEN_EVENTUNREGISTER
-// - MCGEN_EVENTSETINFORMATION
-// - MCGEN_EVENTWRITETRANSFER
-//
-// If the the macro is undefined, the MC implementation will default to the
-// corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is
-// undefined, the EventRegister[MyProviderName] macro will use EventRegister
-// in user mode and will use EtwRegister in kernel mode.
-//
-// To prevent issues from conflicting definitions of these macros, the value
-// of the override macro will be used as a suffix in certain internal function
-// names. Because of this, the override macros must follow certain rules:
-//
-// - The macro must be defined before any MC-generated header is included and
-//   must not be undefined or redefined after any MC-generated header is
-//   included. Different translation units (i.e. different .c or .cpp files)
-//   may set the macros to different values, but within a translation unit
-//   (within a single .c or .cpp file), the macro must be set once and not
-//   changed.
-// - The override must be an object-like macro, not a function-like macro
-//   (i.e. the override macro must not have a parameter list).
-// - The override macro's value must be a simple identifier, i.e. must be
-//   something that starts with a letter or '_' and contains only letters,
-//   numbers, and '_' characters.
-// - If the override macro's value is the name of a second object-like macro,
-//   the second object-like macro must follow the same rules. (The override
-//   macro's value can also be the name of a function-like macro, in which
-//   case the function-like macro does not need to follow the same rules.)
-//
-// For example, the following will cause compile errors:
-//
-//   #define MCGEN_EVENTWRITETRANSFER MyNamespace::MyClass::MyFunction // Value has non-identifier characters (colon).
-//   #define MCGEN_EVENTWRITETRANSFER GetEventWriteFunctionPointer(7)  // Value has non-identifier characters (parentheses).
-//   #define MCGEN_EVENTWRITETRANSFER(h,e,a,r,c,d) EventWrite(h,e,c,d) // Override is defined as a function-like macro.
-//   #define MY_OBJECT_LIKE_MACRO     MyNamespace::MyClass::MyEventWriteFunction
-//   #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // Evaluates to something with non-identifier characters (colon).
-//
-// The following would be ok:
-//
-//   #define MCGEN_EVENTWRITETRANSFER  MyEventWriteFunction1  // OK, suffix will be "MyEventWriteFunction1".
-//   #define MY_OBJECT_LIKE_MACRO      MyEventWriteFunction2
-//   #define MCGEN_EVENTWRITETRANSFER  MY_OBJECT_LIKE_MACRO   // OK, suffix will be "MyEventWriteFunction2".
-//   #define MY_FUNCTION_LIKE_MACRO(h,e,a,r,c,d) MyNamespace::MyClass::MyEventWriteFunction3(h,e,c,d)
-//   #define MCGEN_EVENTWRITETRANSFER  MY_FUNCTION_LIKE_MACRO // OK, suffix will be "MY_FUNCTION_LIKE_MACRO".
-//
-#ifndef MCGEN_EVENTREGISTER
-  #if MCGEN_USE_KERNEL_MODE_APIS
-    #define MCGEN_EVENTREGISTER        EtwRegister
-  #else
-    #define MCGEN_EVENTREGISTER        EventRegister
-  #endif
-#endif // MCGEN_EVENTREGISTER
-#ifndef MCGEN_EVENTUNREGISTER
-  #if MCGEN_USE_KERNEL_MODE_APIS
-    #define MCGEN_EVENTUNREGISTER      EtwUnregister
-  #else
-    #define MCGEN_EVENTUNREGISTER      EventUnregister
-  #endif
-#endif // MCGEN_EVENTUNREGISTER
-#ifndef MCGEN_EVENTSETINFORMATION
-  #if MCGEN_USE_KERNEL_MODE_APIS
-    #define MCGEN_EVENTSETINFORMATION  EtwSetInformation
-  #else
-    #define MCGEN_EVENTSETINFORMATION  EventSetInformation
-  #endif
-#endif // MCGEN_EVENTSETINFORMATION
-#ifndef MCGEN_EVENTWRITETRANSFER
-  #if MCGEN_USE_KERNEL_MODE_APIS
-    #define MCGEN_EVENTWRITETRANSFER   EtwWriteTransfer
-  #else
-    #define MCGEN_EVENTWRITETRANSFER   EventWriteTransfer
-  #endif
-#endif // MCGEN_EVENTWRITETRANSFER
-
-//
-// MCGEN_EVENT_ENABLED macro:
-// Override to control how the EventWrite[EventName] macros determine whether
-// an event is enabled. The default behavior is for EventWrite[EventName] to
-// use the EventEnabled[EventName] macros.
-//
-#ifndef MCGEN_EVENT_ENABLED
-#define MCGEN_EVENT_ENABLED(EventName) EventEnabled##EventName()
-#endif
-
-//
-// MCGEN_EVENT_ENABLED_FORCONTEXT macro:
-// Override to control how the EventWrite[EventName]_ForContext macros
-// determine whether an event is enabled. The default behavior is for
-// EventWrite[EventName]_ForContext to use the
-// EventEnabled[EventName]_ForContext macros.
-//
-#ifndef MCGEN_EVENT_ENABLED_FORCONTEXT
-#define MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, EventName) EventEnabled##EventName##_ForContext(pContext)
-#endif
-
-//
-// MCGEN_ENABLE_CHECK macro:
-// Determines whether the specified event would be considered as enabled
-// based on the state of the specified context. Slightly faster than calling
-// McGenEventEnabled directly.
-//
-#ifndef MCGEN_ENABLE_CHECK
-#define MCGEN_ENABLE_CHECK(Context, Descriptor) (Context.IsEnabled && McGenEventEnabled(&Context, &Descriptor))
-#endif
-
-#if !defined(MCGEN_TRACE_CONTEXT_DEF)
-#define MCGEN_TRACE_CONTEXT_DEF
-// This structure is for use by MC-generated code and should not be used directly.
-typedef struct _MCGEN_TRACE_CONTEXT
-{
-    TRACEHANDLE            RegistrationHandle;
-    TRACEHANDLE            Logger;      // Used as pointer to provider traits.
-    ULONGLONG              MatchAnyKeyword;
-    ULONGLONG              MatchAllKeyword;
-    ULONG                  Flags;
-    ULONG                  IsEnabled;
-    UCHAR                  Level;
-    UCHAR                  Reserve;
-    USHORT                 EnableBitsCount;
-    PULONG                 EnableBitMask;
-    const ULONGLONG*       EnableKeyWords;
-    const UCHAR*           EnableLevel;
-} MCGEN_TRACE_CONTEXT, *PMCGEN_TRACE_CONTEXT;
-#endif // MCGEN_TRACE_CONTEXT_DEF
-
-#if !defined(MCGEN_LEVEL_KEYWORD_ENABLED_DEF)
-#define MCGEN_LEVEL_KEYWORD_ENABLED_DEF
-//
-// Determines whether an event with a given Level and Keyword would be
-// considered as enabled based on the state of the specified context.
-// Note that you may want to use MCGEN_ENABLE_CHECK instead of calling this
-// function directly.
-//
-FORCEINLINE
-BOOLEAN
-McGenLevelKeywordEnabled(
-    _In_ PMCGEN_TRACE_CONTEXT EnableInfo,
-    _In_ UCHAR Level,
-    _In_ ULONGLONG Keyword
-    )
-{
-    //
-    // Check if the event Level is lower than the level at which
-    // the channel is enabled.
-    // If the event Level is 0 or the channel is enabled at level 0,
-    // all levels are enabled.
-    //
-
-    if ((Level <= EnableInfo->Level) || // This also covers the case of Level == 0.
-        (EnableInfo->Level == 0)) {
-
-        //
-        // Check if Keyword is enabled
-        //
-
-        if ((Keyword == (ULONGLONG)0) ||
-            ((Keyword & EnableInfo->MatchAnyKeyword) &&
-             ((Keyword & EnableInfo->MatchAllKeyword) == EnableInfo->MatchAllKeyword))) {
-            return TRUE;
-        }
-    }
-
-    return FALSE;
-}
-#endif // MCGEN_LEVEL_KEYWORD_ENABLED_DEF
-
-#if !defined(MCGEN_EVENT_ENABLED_DEF)
-#define MCGEN_EVENT_ENABLED_DEF
-//
-// Determines whether the specified event would be considered as enabled based
-// on the state of the specified context. Note that you may want to use
-// MCGEN_ENABLE_CHECK instead of calling this function directly.
-//
-FORCEINLINE
-BOOLEAN
-McGenEventEnabled(
-    _In_ PMCGEN_TRACE_CONTEXT EnableInfo,
-    _In_ PCEVENT_DESCRIPTOR EventDescriptor
-    )
-{
-    return McGenLevelKeywordEnabled(EnableInfo, EventDescriptor->Level, EventDescriptor->Keyword);
-}
-#endif // MCGEN_EVENT_ENABLED_DEF
-
-#if !defined(MCGEN_CONTROL_CALLBACK)
-#define MCGEN_CONTROL_CALLBACK
-
-// This function is for use by MC-generated code and should not be used directly.
-DECLSPEC_NOINLINE __inline
-VOID
-__stdcall
-McGenControlCallbackV2(
-    _In_ LPCGUID SourceId,
-    _In_ ULONG ControlCode,
-    _In_ UCHAR Level,
-    _In_ ULONGLONG MatchAnyKeyword,
-    _In_ ULONGLONG MatchAllKeyword,
-    _In_opt_ PEVENT_FILTER_DESCRIPTOR FilterData,
-    _Inout_opt_ PVOID CallbackContext
-    )
-/*++
-
-Routine Description:
-
-    This is the notification callback for Windows Vista and later.
-
-Arguments:
-
-    SourceId - The GUID that identifies the session that enabled the provider.
-
-    ControlCode - The parameter indicates whether the provider
-                  is being enabled or disabled.
-
-    Level - The level at which the event is enabled.
-
-    MatchAnyKeyword - The bitmask of keywords that the provider uses to
-                      determine the category of events that it writes.
-
-    MatchAllKeyword - This bitmask additionally restricts the category
-                      of events that the provider writes.
-
-    FilterData - The provider-defined data.
-
-    CallbackContext - The context of the callback that is defined when the provider
-                      called EtwRegister to register itself.
-
-Remarks:
-
-    ETW calls this function to notify provider of enable/disable
-
---*/
-{
-    PMCGEN_TRACE_CONTEXT Ctx = (PMCGEN_TRACE_CONTEXT)CallbackContext;
-    ULONG Ix;
-#ifndef MCGEN_PRIVATE_ENABLE_CALLBACK_V2
-    UNREFERENCED_PARAMETER(SourceId);
-    UNREFERENCED_PARAMETER(FilterData);
-#endif
-
-    if (Ctx == NULL) {
-        return;
-    }
-
-    switch (ControlCode) {
-
-        case EVENT_CONTROL_CODE_ENABLE_PROVIDER:
-            Ctx->Level = Level;
-            Ctx->MatchAnyKeyword = MatchAnyKeyword;
-            Ctx->MatchAllKeyword = MatchAllKeyword;
-            Ctx->IsEnabled = EVENT_CONTROL_CODE_ENABLE_PROVIDER;
-
-            for (Ix = 0; Ix < Ctx->EnableBitsCount; Ix += 1) {
-                if (McGenLevelKeywordEnabled(Ctx, Ctx->EnableLevel[Ix], Ctx->EnableKeyWords[Ix]) != FALSE) {
-                    Ctx->EnableBitMask[Ix >> 5] |= (1 << (Ix % 32));
-                } else {
-                    Ctx->EnableBitMask[Ix >> 5] &= ~(1 << (Ix % 32));
-                }
-            }
-            break;
-
-        case EVENT_CONTROL_CODE_DISABLE_PROVIDER:
-            Ctx->IsEnabled = EVENT_CONTROL_CODE_DISABLE_PROVIDER;
-            Ctx->Level = 0;
-            Ctx->MatchAnyKeyword = 0;
-            Ctx->MatchAllKeyword = 0;
-            if (Ctx->EnableBitsCount > 0) {
-#pragma warning(suppress: 26451) // Arithmetic overflow cannot occur, no matter the value of EnableBitCount
-                RtlZeroMemory(Ctx->EnableBitMask, (((Ctx->EnableBitsCount - 1) / 32) + 1) * sizeof(ULONG));
-            }
-            break;
-
-        default:
-            break;
-    }
-
-#ifdef MCGEN_PRIVATE_ENABLE_CALLBACK_V2
-    //
-    // Call user defined callback
-    //
-    MCGEN_PRIVATE_ENABLE_CALLBACK_V2(
-        SourceId,
-        ControlCode,
-        Level,
-        MatchAnyKeyword,
-        MatchAllKeyword,
-        FilterData,
-        CallbackContext
-        );
-#endif // MCGEN_PRIVATE_ENABLE_CALLBACK_V2
-
-    return;
-}
-
-#endif // MCGEN_CONTROL_CALLBACK
-
-#ifndef _mcgen_PENABLECALLBACK
-  #if MCGEN_USE_KERNEL_MODE_APIS
-    #define _mcgen_PENABLECALLBACK      PETWENABLECALLBACK
-  #else
-    #define _mcgen_PENABLECALLBACK      PENABLECALLBACK
-  #endif
-#endif // _mcgen_PENABLECALLBACK
-
-#if !defined(_mcgen_PASTE2)
-// This macro is for use by MC-generated code and should not be used directly.
-#define _mcgen_PASTE2(a, b) _mcgen_PASTE2_imp(a, b)
-#define _mcgen_PASTE2_imp(a, b) a##b
-#endif // _mcgen_PASTE2
-
-#if !defined(_mcgen_PASTE3)
-// This macro is for use by MC-generated code and should not be used directly.
-#define _mcgen_PASTE3(a, b, c) _mcgen_PASTE3_imp(a, b, c)
-#define _mcgen_PASTE3_imp(a, b, c) a##b##_##c
-#endif // _mcgen_PASTE3
-
-//
-// Macro validation
-//
-
-// Validate MCGEN_EVENTREGISTER:
-
-// Trigger an error if MCGEN_EVENTREGISTER is not an unqualified (simple) identifier:
-struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER);
-
-// Trigger an error if MCGEN_EVENTREGISTER is redefined:
-typedef struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER)
-    MCGEN_EVENTREGISTER_must_not_be_redefined_between_headers;
-
-// Trigger an error if MCGEN_EVENTREGISTER is defined as a function-like macro:
-typedef void MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTREGISTER;
-typedef int _mcgen_PASTE2(MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTREGISTER);
-
-// Validate MCGEN_EVENTUNREGISTER:
-
-// Trigger an error if MCGEN_EVENTUNREGISTER is not an unqualified (simple) identifier:
-struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER);
-
-// Trigger an error if MCGEN_EVENTUNREGISTER is redefined:
-typedef struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER)
-    MCGEN_EVENTUNREGISTER_must_not_be_redefined_between_headers;
-
-// Trigger an error if MCGEN_EVENTUNREGISTER is defined as a function-like macro:
-typedef void MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTUNREGISTER;
-typedef int _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTUNREGISTER);
-
-// Validate MCGEN_EVENTSETINFORMATION:
-
-// Trigger an error if MCGEN_EVENTSETINFORMATION is not an unqualified (simple) identifier:
-struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION);
-
-// Trigger an error if MCGEN_EVENTSETINFORMATION is redefined:
-typedef struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION)
-    MCGEN_EVENTSETINFORMATION_must_not_be_redefined_between_headers;
-
-// Trigger an error if MCGEN_EVENTSETINFORMATION is defined as a function-like macro:
-typedef void MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_MCGEN_EVENTSETINFORMATION;
-typedef int _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_, MCGEN_EVENTSETINFORMATION);
-
-// Validate MCGEN_EVENTWRITETRANSFER:
-
-// Trigger an error if MCGEN_EVENTWRITETRANSFER is not an unqualified (simple) identifier:
-struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER);
-
-// Trigger an error if MCGEN_EVENTWRITETRANSFER is redefined:
-typedef struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER)
-    MCGEN_EVENTWRITETRANSFER_must_not_be_redefined_between_headers;;
-
-// Trigger an error if MCGEN_EVENTWRITETRANSFER is defined as a function-like macro:
-typedef void MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_MCGEN_EVENTWRITETRANSFER;
-typedef int _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_, MCGEN_EVENTWRITETRANSFER);
-
-#ifndef McGenEventWrite_def
-#define McGenEventWrite_def
-
-// This macro is for use by MC-generated code and should not be used directly.
-#define McGenEventWrite _mcgen_PASTE2(McGenEventWrite_, MCGEN_EVENTWRITETRANSFER)
-
-// This function is for use by MC-generated code and should not be used directly.
-DECLSPEC_NOINLINE __inline
-ULONG __stdcall
-McGenEventWrite(
-    _In_ PMCGEN_TRACE_CONTEXT Context,
-    _In_ PCEVENT_DESCRIPTOR Descriptor,
-    _In_opt_ LPCGUID ActivityId,
-    _In_range_(1, 128) ULONG EventDataCount,
-    _Pre_cap_(EventDataCount) EVENT_DATA_DESCRIPTOR* EventData
-    )
-{
-    const USHORT UNALIGNED* Traits;
-
-    // Some customized MCGEN_EVENTWRITETRANSFER macros might ignore ActivityId.
-    UNREFERENCED_PARAMETER(ActivityId);
-
-    Traits = (const USHORT UNALIGNED*)(UINT_PTR)Context->Logger;
-
-    if (Traits == NULL) {
-        EventData[0].Ptr = 0;
-        EventData[0].Size = 0;
-        EventData[0].Reserved = 0;
-    } else {
-        EventData[0].Ptr = (ULONG_PTR)Traits;
-        EventData[0].Size = *Traits;
-        EventData[0].Reserved = 2; // EVENT_DATA_DESCRIPTOR_TYPE_PROVIDER_METADATA
-    }
-
-    return MCGEN_EVENTWRITETRANSFER(
-        Context->RegistrationHandle,
-        Descriptor,
-        ActivityId,
-        NULL,
-        EventDataCount,
-        EventData);
-}
-#endif // McGenEventWrite_def
-
-#if !defined(McGenEventRegisterUnregister)
-#define McGenEventRegisterUnregister
-
-// This macro is for use by MC-generated code and should not be used directly.
-#define McGenEventRegister _mcgen_PASTE2(McGenEventRegister_, MCGEN_EVENTREGISTER)
-
-#pragma warning(push)
-#pragma warning(disable:6103)
-// This function is for use by MC-generated code and should not be used directly.
-DECLSPEC_NOINLINE __inline
-ULONG __stdcall
-McGenEventRegister(
-    _In_ LPCGUID ProviderId,
-    _In_opt_ _mcgen_PENABLECALLBACK EnableCallback,
-    _In_opt_ PVOID CallbackContext,
-    _Inout_ PREGHANDLE RegHandle
-    )
-/*++
-
-Routine Description:
-
-    This function registers the provider with ETW.
-
-Arguments:
-
-    ProviderId - Provider ID to register with ETW.
-
-    EnableCallback - Callback to be used.
-
-    CallbackContext - Context for the callback.
-
-    RegHandle - Pointer to registration handle.
-
-Remarks:
-
-    Should not be called if the provider is already registered (i.e. should not
-    be called if *RegHandle != 0). Repeatedly registering a provider is a bug
-    and may indicate a race condition. However, for compatibility with previous
-    behavior, this function will return SUCCESS in this case.
-
---*/
-{
-    ULONG Error;
-
-    if (*RegHandle != 0)
-    {
-        Error = 0; // ERROR_SUCCESS
-    }
-    else
-    {
-        Error = MCGEN_EVENTREGISTER(ProviderId, EnableCallback, CallbackContext, RegHandle);
-    }
-
-    return Error;
-}
-#pragma warning(pop)
-
-// This macro is for use by MC-generated code and should not be used directly.
-#define McGenEventUnregister _mcgen_PASTE2(McGenEventUnregister_, MCGEN_EVENTUNREGISTER)
-
-// This function is for use by MC-generated code and should not be used directly.
-DECLSPEC_NOINLINE __inline
-ULONG __stdcall
-McGenEventUnregister(_Inout_ PREGHANDLE RegHandle)
-/*++
-
-Routine Description:
-
-    Unregister from ETW and set *RegHandle = 0.
-
-Arguments:
-
-    RegHandle - the pointer to the provider registration handle
-
-Remarks:
-
-    If provider has not been registered (i.e. if *RegHandle == 0),
-    return SUCCESS. It is safe to call McGenEventUnregister even if the
-    call to McGenEventRegister returned an error.
-
---*/
-{
-    ULONG Error;
-
-    if(*RegHandle == 0)
-    {
-        Error = 0; // ERROR_SUCCESS
-    }
-    else
-    {
-        Error = MCGEN_EVENTUNREGISTER(*RegHandle);
-        *RegHandle = (REGHANDLE)0;
-    }
-
-    return Error;
-}
-
-#endif // McGenEventRegisterUnregister
-
-#ifndef _mcgen_EVENT_BIT_SET
-  #if defined(_M_IX86) || defined(_M_X64)
-    // This macro is for use by MC-generated code and should not be used directly.
-    #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((((const unsigned char*)EnableBits)[BitPosition >> 3] & (1u << (BitPosition & 7))) != 0)
-  #else // CPU type
-    // This macro is for use by MC-generated code and should not be used directly.
-    #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((EnableBits[BitPosition >> 5] & (1u << (BitPosition & 31))) != 0)
-  #endif // CPU type
-#endif // _mcgen_EVENT_BIT_SET
-
-#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION
-
-//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-// Provider "microsoft-windows-mimalloc" event count 2
-//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-// Provider GUID = 138f4dbb-ee04-4899-aa0a-572ad4475779
-EXTERN_C __declspec(selectany) const GUID ETW_MI_Provider = {0x138f4dbb, 0xee04, 0x4899, {0xaa, 0x0a, 0x57, 0x2a, 0xd4, 0x47, 0x57, 0x79}};
-
-#ifndef ETW_MI_Provider_Traits
-#define ETW_MI_Provider_Traits NULL
-#endif // ETW_MI_Provider_Traits
-
-//
-// Event Descriptors
-//
-EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_ALLOC = {0x64, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};
-#define ETW_MI_ALLOC_value 0x64
-EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_FREE = {0x65, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};
-#define ETW_MI_FREE_value 0x65
-
-//
-// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:
-// Define this macro to have the compiler skip the generated functions in this
-// header.
-//
-#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION
-
-//
-// Event Enablement Bits
-// These variables are for use by MC-generated code and should not be used directly.
-//
-EXTERN_C __declspec(selectany) DECLSPEC_CACHEALIGN ULONG microsoft_windows_mimallocEnableBits[1];
-EXTERN_C __declspec(selectany) const ULONGLONG microsoft_windows_mimallocKeywords[1] = {0x0};
-EXTERN_C __declspec(selectany) const unsigned char microsoft_windows_mimallocLevels[1] = {4};
-
-//
-// Provider context
-//
-EXTERN_C __declspec(selectany) MCGEN_TRACE_CONTEXT ETW_MI_Provider_Context = {0, (ULONG_PTR)ETW_MI_Provider_Traits, 0, 0, 0, 0, 0, 0, 1, microsoft_windows_mimallocEnableBits, microsoft_windows_mimallocKeywords, microsoft_windows_mimallocLevels};
-
-//
-// Provider REGHANDLE
-//
-#define microsoft_windows_mimallocHandle (ETW_MI_Provider_Context.RegistrationHandle)
-
-//
-// This macro is set to 0, indicating that the EventWrite[Name] macros do not
-// have an Activity parameter. This is controlled by the -km and -um options.
-//
-#define ETW_MI_Provider_EventWriteActivity 0
-
-//
-// Register with ETW using the control GUID specified in the manifest.
-// Invoke this macro during module initialization (i.e. program startup,
-// DLL process attach, or driver load) to initialize the provider.
-// Note that if this function returns an error, the error means that
-// will not work, but no action needs to be taken -- even if EventRegister
-// returns an error, it is generally safe to use EventWrite and
-// EventUnregister macros (they will be no-ops if EventRegister failed).
-//
-#ifndef EventRegistermicrosoft_windows_mimalloc
-#define EventRegistermicrosoft_windows_mimalloc() McGenEventRegister(&ETW_MI_Provider, McGenControlCallbackV2, &ETW_MI_Provider_Context, &microsoft_windows_mimallocHandle)
-#endif
-
-//
-// Register with ETW using a specific control GUID (i.e. a GUID other than what
-// is specified in the manifest). Advanced scenarios only.
-//
-#ifndef EventRegisterByGuidmicrosoft_windows_mimalloc
-#define EventRegisterByGuidmicrosoft_windows_mimalloc(Guid) McGenEventRegister(&(Guid), McGenControlCallbackV2, &ETW_MI_Provider_Context, &microsoft_windows_mimallocHandle)
-#endif
-
-//
-// Unregister with ETW and close the provider.
-// Invoke this macro during module shutdown (i.e. program exit, DLL process
-// detach, or driver unload) to unregister the provider.
-// Note that you MUST call EventUnregister before DLL or driver unload
-// (not optional): failure to unregister a provider before DLL or driver unload
-// will result in crashes.
-//
-#ifndef EventUnregistermicrosoft_windows_mimalloc
-#define EventUnregistermicrosoft_windows_mimalloc() McGenEventUnregister(&microsoft_windows_mimallocHandle)
-#endif
-
-//
-// MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION macro:
-// Define this macro to enable support for caller-allocated provider context.
-//
-#ifdef MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION
-
-//
-// Advanced scenarios: Caller-allocated provider context.
-// Use when multiple differently-configured provider handles are needed,
-// e.g. for container-aware drivers, one context per container.
-//
-// Usage:
-//
-// - Caller enables the feature before including this header, e.g.
-//   #define MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION 1
-// - Caller allocates memory, e.g. pContext = malloc(sizeof(McGenContext_microsoft_windows_mimalloc));
-// - Caller registers the provider, e.g. EventRegistermicrosoft_windows_mimalloc_ForContext(pContext);
-// - Caller writes events, e.g. EventWriteMyEvent_ForContext(pContext, ...);
-// - Caller unregisters, e.g. EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext);
-// - Caller frees memory, e.g. free(pContext);
-//
-
-typedef struct tagMcGenContext_microsoft_windows_mimalloc {
-    // The fields of this structure are subject to change and should
-    // not be accessed directly. To access the provider's REGHANDLE,
-    // use microsoft_windows_mimallocHandle_ForContext(pContext).
-    MCGEN_TRACE_CONTEXT Context;
-    ULONG EnableBits[1];
-} McGenContext_microsoft_windows_mimalloc;
-
-#define EventRegistermicrosoft_windows_mimalloc_ForContext(pContext)             _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&ETW_MI_Provider, pContext)
-#define EventRegisterByGuidmicrosoft_windows_mimalloc_ForContext(Guid, pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&(Guid), pContext)
-#define EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext)           McGenEventUnregister(&(pContext)->Context.RegistrationHandle)
-
-//
-// Provider REGHANDLE for caller-allocated context.
-//
-#define microsoft_windows_mimallocHandle_ForContext(pContext) ((pContext)->Context.RegistrationHandle)
-
-// This function is for use by MC-generated code and should not be used directly.
-// Initialize and register the caller-allocated context.
-__inline
-ULONG __stdcall
-_mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(
-    _In_ LPCGUID pProviderId,
-    _Out_ McGenContext_microsoft_windows_mimalloc* pContext)
-{
-    RtlZeroMemory(pContext, sizeof(*pContext));
-    pContext->Context.Logger = (ULONG_PTR)ETW_MI_Provider_Traits;
-    pContext->Context.EnableBitsCount = 1;
-    pContext->Context.EnableBitMask = pContext->EnableBits;
-    pContext->Context.EnableKeyWords = microsoft_windows_mimallocKeywords;
-    pContext->Context.EnableLevel = microsoft_windows_mimallocLevels;
-    return McGenEventRegister(
-        pProviderId,
-        McGenControlCallbackV2,
-        &pContext->Context,
-        &pContext->Context.RegistrationHandle);
-}
-
-// This function is for use by MC-generated code and should not be used directly.
-// Trigger a compile error if called with the wrong parameter type.
-FORCEINLINE
-_Ret_ McGenContext_microsoft_windows_mimalloc*
-_mcgen_CheckContextType_microsoft_windows_mimalloc(_In_ McGenContext_microsoft_windows_mimalloc* pContext)
-{
-    return pContext;
-}
-
-#endif // MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION
-
-//
-// Enablement check macro for event "ETW_MI_ALLOC"
-//
-#define EventEnabledETW_MI_ALLOC() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)
-#define EventEnabledETW_MI_ALLOC_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)
-
-//
-// Event write macros for event "ETW_MI_ALLOC"
-//
-#define EventWriteETW_MI_ALLOC(Address, Size) \
-        MCGEN_EVENT_ENABLED(ETW_MI_ALLOC) \
-        ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) : 0
-#define EventWriteETW_MI_ALLOC_AssumeEnabled(Address, Size) \
-        _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size)
-#define EventWriteETW_MI_ALLOC_ForContext(pContext, Address, Size) \
-        MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_ALLOC) \
-        ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&(pContext)->Context, &ETW_MI_ALLOC, Address, Size) : 0
-#define EventWriteETW_MI_ALLOC_ForContextAssumeEnabled(pContext, Address, Size) \
-        _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_ALLOC, Address, Size)
-
-// This macro is for use by MC-generated code and should not be used directly.
-#define _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)
-
-//
-// Enablement check macro for event "ETW_MI_FREE"
-//
-#define EventEnabledETW_MI_FREE() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)
-#define EventEnabledETW_MI_FREE_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)
-
-//
-// Event write macros for event "ETW_MI_FREE"
-//
-#define EventWriteETW_MI_FREE(Address, Size) \
-        MCGEN_EVENT_ENABLED(ETW_MI_FREE) \
-        ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) : 0
-#define EventWriteETW_MI_FREE_AssumeEnabled(Address, Size) \
-        _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size)
-#define EventWriteETW_MI_FREE_ForContext(pContext, Address, Size) \
-        MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_FREE) \
-        ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&(pContext)->Context, &ETW_MI_FREE, Address, Size) : 0
-#define EventWriteETW_MI_FREE_ForContextAssumeEnabled(pContext, Address, Size) \
-        _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_FREE, Address, Size)
-
-// This macro is for use by MC-generated code and should not be used directly.
-#define _mcgen_TEMPLATE_FOR_ETW_MI_FREE _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)
-
-#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION
-
-//
-// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:
-// Define this macro to have the compiler skip the generated functions in this
-// header.
-//
-#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION
-
-//
-// Template Functions
-//
-
-//
-// Function for template "ETW_CUSTOM_HEAP_ALLOC_DATA" (and possibly others).
-// This function is for use by MC-generated code and should not be used directly.
-//
-#ifndef McTemplateU0xx_def
-#define McTemplateU0xx_def
-ETW_INLINE
-ULONG
-_mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)(
-    _In_ PMCGEN_TRACE_CONTEXT Context,
-    _In_ PCEVENT_DESCRIPTOR Descriptor,
-    _In_ const unsigned __int64  _Arg0,
-    _In_ const unsigned __int64  _Arg1
-    )
-{
-#define McTemplateU0xx_ARGCOUNT 2
-
-    EVENT_DATA_DESCRIPTOR EventData[McTemplateU0xx_ARGCOUNT + 1];
-
-    EventDataDescCreate(&EventData[1],&_Arg0, sizeof(const unsigned __int64)  );
-
-    EventDataDescCreate(&EventData[2],&_Arg1, sizeof(const unsigned __int64)  );
-
-    return McGenEventWrite(Context, Descriptor, NULL, McTemplateU0xx_ARGCOUNT + 1, EventData);
-}
-#endif // McTemplateU0xx_def
-
-#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION
-
-#if defined(__cplusplus)
-}
-#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.man b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/etw.man
deleted file mode 100644
index cfd1f8a9eaacd50af63f1e28f9540aa88c20f90c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3926
zcmeH~T~8B16o${WiT`2c+NGc<*i;EYh$d8xl<0*CS-Mb~v<qPu>PP(R>hsQY*nU!q
z$b})3?##}d?{nTW+uy%xwr*doYaNU1!Vc}sa%<a(W%kk*Y}poVi8<mVvnKl~r<t|f
z>7F%g+hVAmL$hwL?4dodnmuAKhUXm0)X9|WHj>XRahh@~SWDIkbduX;B#u6^Q>@U=
zDO8U+KXa0*th&%f*z^Udh4ol@uE=Q&`emUsh_CA`FOXgI{i-`XZ9C#bR1yBm=PJ*p
z9kVN$J6O;h;8HY>p)RnhY8A#Hb?z)_!y(Iaen(I)@-9CrSSp(;_Jn9I*$S&ATjP1?
zVxB>pV@LVsy;^jZr7r$HNAm06TcUiI`l@~F$Mt$E%Shfd3O+h1vFhR9a8yQZ@wpne
zr3bI-p=VEdo{)#uWxSVJeYQF|-5tnq>~f+CP~A0&{v=(up=ngEDl>5!$EDwPRaNjW
zXj|wbG$OwmwaW-hMvBK%pbm3wpic7<iGe1}wLX@G{?G|Bd>1O^dzbxT%*13+SP9h-
zI~rA5hapTVnk|qmiIVYy{__+x9f7OV4j3`g4;{{8_SWnLBSu2PUc%~`t%Ae^>J`SS
zdtZg-r<0xAH*_ALtK;Nv(d9nbKK1jK=Ld)I(jQrKhBjgToR#Wm8{0a}@6ZuEO<gbQ
zB3m5V%^3-v%%-25hY;g1&y$zH6XrqQ2)nL|zy>(lvG=y=Jh{M!4!-$(E)!vYUrf47
z<hkOaGap-@>nf4W$e&QFOovV_$>J%X*KN>oXI@jt%BJms>IU}I$<7<wvJG;y&ofxL
zZ?Ac4@EfiF;Qh3@Hq?u*mxUl}o``PSN9^9363^xS{`lRzaEI{>Hr{PChchs%+mx{%
zt(fa_PM4r63?1h#YOk~;byc5`>%q>sLHA1gohNq{qOREhxu<<B25jv)bQOG~tvEXM
zsbYvycWRYgeO<KaNEs4R@T0H57CiG+erb6HDME8{V+nzO*!Nwm{Em&P|NqBp)%s}N
z&KLRG82>y~`}YVhGe0?R_ceQ8vt^BpSNp6kErj_0hUNFyWQ1IOZ|GEgWBPwYFLgFu
Oo!*uqEBu%Ae18C_t1cS=

diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/prim.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/prim.c
deleted file mode 100644
index 51ad3104..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/prim.c
+++ /dev/null
@@ -1,1124 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-// This file is included in `src/prim/prim.c`
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"
-#include <stdio.h>   // fputs, stderr
-#include <stdlib.h>  // atexit
-
-// xbox has no console IO
-#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM)
-#define MI_HAS_CONSOLE_IO
-#endif
-
-//---------------------------------------------
-// Dynamically bind Windows API points for portability
-//---------------------------------------------
-
-#if defined(_MSC_VER)
-#pragma warning(disable:4996)   // don't use GetVersionExW
-#endif
-
-static DWORD win_major_version = 6;
-static DWORD win_minor_version = 0;
-
-// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
-// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
-// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB)
-// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's.
-typedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E {
-  MiMemExtendedParameterInvalidType = 0,
-  MiMemExtendedParameterAddressRequirements,
-  MiMemExtendedParameterNumaNode,
-  MiMemExtendedParameterPartitionHandle,
-  MiMemExtendedParameterUserPhysicalHandle,
-  MiMemExtendedParameterAttributeFlags,
-  MiMemExtendedParameterMax
-} MI_MEM_EXTENDED_PARAMETER_TYPE;
-
-typedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S {
-  struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type;
-  union  { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg;
-} MI_MEM_EXTENDED_PARAMETER;
-
-typedef struct MI_MEM_ADDRESS_REQUIREMENTS_S {
-  PVOID  LowestStartingAddress;
-  PVOID  HighestEndingAddress;
-  SIZE_T Alignment;
-} MI_MEM_ADDRESS_REQUIREMENTS;
-
-#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE   0x00000010
-
-#include <winternl.h>
-typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);
-typedef LONG  (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);  // avoid NTSTATUS as it is not defined on xbox (pr #1084)
-static PVirtualAlloc2 pVirtualAlloc2 = NULL;
-static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;
-
-// Similarly, GetNumaProcessorNodeEx is only supported since Windows 7  (and GetNumaNodeProcessorMask is not supported on xbox)
-typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER;
-
-typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber);
-typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber);
-typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask);
-typedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber);
-typedef BOOL (__stdcall* PGetNumaNodeProcessorMask)(UCHAR Node, PULONGLONG ProcessorMask);
-typedef BOOL (__stdcall* PGetNumaHighestNodeNumber)(PULONG Node);
-static PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL;
-static PGetNumaProcessorNodeEx      pGetNumaProcessorNodeEx = NULL;
-static PGetNumaNodeProcessorMaskEx  pGetNumaNodeProcessorMaskEx = NULL;
-static PGetNumaProcessorNode        pGetNumaProcessorNode = NULL;
-static PGetNumaNodeProcessorMask    pGetNumaNodeProcessorMask = NULL;
-static PGetNumaHighestNodeNumber    pGetNumaHighestNodeNumber = NULL;
-
-// Not available on xbox
-typedef SIZE_T(__stdcall* PGetLargePageMinimum)(VOID);
-static PGetLargePageMinimum pGetLargePageMinimum = NULL;
-
-// Available after Windows XP
-typedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalMemoryInKilobytes );
-typedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation);
-
-
-
-//---------------------------------------------
-// Enable large page support dynamically (if possible)
-//---------------------------------------------
-
-static bool win_enable_large_os_pages_once(size_t* large_page_size)
-{
-  if (pGetLargePageMinimum==NULL) return false;  // no large page support (xbox etc.)
-
-  // Try to see if large OS pages are supported
-  // To use large pages on Windows, we first need access permission
-  // Set "Lock pages in memory" permission in the group policy editor
-  // <https://devblogs.microsoft.com/oldnewthing/20110128-00/?p=11643>
-  unsigned long err = 0;
-  HANDLE token = NULL;
-  BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
-  if (ok) {
-    TOKEN_PRIVILEGES tp;
-    ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid);
-    if (ok) {
-      tp.PrivilegeCount = 1;
-      tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
-      ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);
-      if (ok) {
-        err = GetLastError();
-        ok = (err == ERROR_SUCCESS);
-        if (ok && large_page_size != NULL && pGetLargePageMinimum != NULL) {
-          *large_page_size = (*pGetLargePageMinimum)();
-        }
-      }
-    }
-    CloseHandle(token);
-  }
-  if (!ok) {
-    if (err == 0) err = GetLastError();
-    _mi_warning_message("cannot enable large OS page support, error %lu\n", err);
-  }
-  return (ok!=0);
-}
-
-static bool win_enable_large_os_pages(size_t* large_page_size) {
-  mi_atomic_do_once {
-    win_enable_large_os_pages_once(large_page_size);
-  }
-  return (_mi_os_large_page_size() > 0);
-}
-
-
-//---------------------------------------------
-// Initialize
-//---------------------------------------------
-
-static DWORD win_allocation_granularity = 64*MI_KiB;
-
-void _mi_prim_mem_init( mi_os_mem_config_t* config )
-{
-  config->has_overcommit = false;
-  config->has_partial_free = false;
-  config->has_virtual_reserve = true;
-
-  // get the page size
-  SYSTEM_INFO si; _mi_memzero_var(si);
-  GetSystemInfo(&si);
-  if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; }
-  if (si.dwAllocationGranularity > 0) {
-    config->alloc_granularity = si.dwAllocationGranularity;
-    win_allocation_granularity = si.dwAllocationGranularity;
-  }
-  // get virtual address bits
-  if ((uintptr_t)si.lpMaximumApplicationAddress > 0) {
-    const size_t vbits = MI_SIZE_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);
-    config->virtual_address_bits = vbits;
-  }
-
-  // get the VirtualAlloc2 function
-  HINSTANCE hDll = LoadLibrary(TEXT("kernelbase.dll"));
-  if (hDll != NULL) {
-    // use VirtualAlloc2FromApp if possible as it is available to Windows store apps
-    pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp");
-    if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2");
-    FreeLibrary(hDll);
-  }
-  // NtAllocateVirtualMemoryEx is used for huge page allocation
-  hDll = LoadLibrary(TEXT("ntdll.dll"));
-  if (hDll != NULL) {
-    pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx");
-    FreeLibrary(hDll);
-  }
-  // Try to use Win7+ numa API
-  hDll = LoadLibrary(TEXT("kernel32.dll"));
-  if (hDll != NULL) {
-    pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx");
-    pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx");
-    pGetNumaNodeProcessorMaskEx = (PGetNumaNodeProcessorMaskEx)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMaskEx");
-    pGetNumaProcessorNode = (PGetNumaProcessorNode)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNode");
-    pGetNumaNodeProcessorMask = (PGetNumaNodeProcessorMask)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMask");
-    pGetNumaHighestNodeNumber = (PGetNumaHighestNodeNumber)(void (*)(void))GetProcAddress(hDll, "GetNumaHighestNodeNumber");
-    pGetLargePageMinimum = (PGetLargePageMinimum)(void (*)(void))GetProcAddress(hDll, "GetLargePageMinimum");
-    // Get physical memory (not available on XP, so check dynamically)
-    PGetPhysicallyInstalledSystemMemory pGetPhysicallyInstalledSystemMemory = (PGetPhysicallyInstalledSystemMemory)(void (*)(void))GetProcAddress(hDll,"GetPhysicallyInstalledSystemMemory");
-    if (pGetPhysicallyInstalledSystemMemory != NULL) {
-      ULONGLONG memInKiB = 0;
-      if ((*pGetPhysicallyInstalledSystemMemory)(&memInKiB)) {
-        if (memInKiB > 0 && memInKiB <= SIZE_MAX) {
-          config->physical_memory_in_kib = (size_t)memInKiB;
-        }
-      }
-    }
-    // Get Windows version
-    PGetVersionExW pGetVersionExW = (PGetVersionExW)(void (*)(void))GetProcAddress(hDll, "GetVersionExW");
-    if (pGetVersionExW != NULL) {
-      OSVERSIONINFOW version; _mi_memzero_var(version);
-      version.dwOSVersionInfoSize = sizeof(version);
-      if ((*pGetVersionExW)(&version)) {
-        win_major_version = version.dwMajorVersion;
-        win_minor_version = version.dwMinorVersion;
-      }
-    }
-    FreeLibrary(hDll);
-  }
-  // Enable large/huge OS page support?
-  if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
-    win_enable_large_os_pages(&config->large_page_size);
-  }
-}
-
-
-//---------------------------------------------
-// Free
-//---------------------------------------------
-
-int _mi_prim_free(void* addr, size_t size ) {
-  MI_UNUSED(size);
-  DWORD errcode = 0;
-  bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);
-  if (err) { errcode = GetLastError(); }
-  if (errcode == ERROR_INVALID_ADDRESS) {
-    // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside
-    // the memory region returned by VirtualAlloc; in that case we need to free using
-    // the start of the region.
-    MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info);
-    VirtualQuery(addr, &info, sizeof(info));
-    if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) {
-      errcode = 0;
-      err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0);
-      if (err) { errcode = GetLastError(); }
-    }
-  }
-  return (int)errcode;
-}
-
-
-//---------------------------------------------
-// VirtualAlloc
-//---------------------------------------------
-
-static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) {
-  #if (MI_INTPTR_SIZE >= 8)
-  // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations
-  if (addr == NULL) {
-    void* hint = _mi_os_get_aligned_hint(try_alignment,size);
-    if (hint != NULL) {
-      void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE);
-      if (p != NULL) return p;
-      _mi_verbose_message("warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), hint, try_alignment, flags);
-      // fall through on error
-    }
-  }
-  #endif
-  // on modern Windows try use VirtualAlloc2 for aligned allocation
-  if (addr == NULL && try_alignment > win_allocation_granularity && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
-    MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };
-    reqs.Alignment = try_alignment;
-    MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };
-    param.Type.Type = MiMemExtendedParameterAddressRequirements;
-    param.Arg.Pointer = &reqs;
-    void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, &param, 1);
-    if (p != NULL) return p;
-    _mi_warning_message("unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags);
-    // fall through on error
-  }
-  // last resort
-  return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
-}
-
-static bool win_is_out_of_memory_error(DWORD err) {
-  switch (err) {
-    case ERROR_COMMITMENT_MINIMUM:
-    case ERROR_COMMITMENT_LIMIT:
-    case ERROR_PAGEFILE_QUOTA:
-    case ERROR_NOT_ENOUGH_MEMORY:
-      return true;
-    default:
-      return false;
-  }
-}
-
-static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {
-  long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000);  // at most 2 seconds
-  if (max_retry_msecs == 1) { max_retry_msecs = 100; }  // if one sets the option to "true"
-  for (long tries = 1; tries <= 10; tries++) {          // try at most 10 times (=2200ms)
-    void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags);
-    if (p != NULL) {
-      // success, return the address
-      return p;
-    }
-    else if (max_retry_msecs > 0 && (try_alignment <= 8*MI_MiB) &&
-              (flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 &&
-              win_is_out_of_memory_error(GetLastError())) {
-      // if committing regular memory and being out-of-memory,
-      // keep trying for a bit in case memory frees up after all. See issue #894
-      _mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags);
-      long sleep_msecs = tries*40;  // increasing waits
-      if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; }
-      max_retry_msecs -= sleep_msecs;
-      Sleep(sleep_msecs);
-    }
-    else {
-      // otherwise return with an error
-      break;
-    }
-  }
-  return NULL;
-}
-
-static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {
-  mi_assert_internal(!(large_only && !allow_large));
-  static _Atomic(size_t) large_page_try_ok; // = 0;
-  void* p = NULL;
-  // Try to allocate large OS pages (2MiB) if allowed or required.
-  if ((large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))
-      && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0)
-  {
-    size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
-    if (!large_only && try_ok > 0) {
-      // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.
-      // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times.
-      mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);
-    }
-    else {
-      // large OS pages must always reserve and commit.
-      *is_large = true;
-      p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES);
-      if (large_only) return p;
-      // fall back to non-large page allocation on error (`p == NULL`).
-      if (p == NULL) {
-        mi_atomic_store_release(&large_page_try_ok,10UL);  // on error, don't try again for the next N allocations
-      }
-    }
-  }
-  // Fall back to regular page allocation
-  if (p == NULL) {
-    *is_large = ((flags&MEM_LARGE_PAGES) != 0);
-    p = win_virtual_alloc_prim(addr, size, try_alignment, flags);
-  }
-  //if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); }
-  return p;
-}
-
-int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
-  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
-  mi_assert_internal(commit || !allow_large);
-  mi_assert_internal(try_alignment > 0);
-  *is_zero = true;
-  int flags = MEM_RESERVE;
-  if (commit) { flags |= MEM_COMMIT; }
-  *addr = win_virtual_alloc(hint_addr, size, try_alignment, flags, false, allow_large, is_large);
-  return (*addr != NULL ? 0 : (int)GetLastError());
-}
-
-
-//---------------------------------------------
-// Commit/Reset/Protect
-//---------------------------------------------
-#ifdef _MSC_VER
-#pragma warning(disable:6250)   // suppress warning calling VirtualFree without MEM_RELEASE (for decommit)
-#endif
-
-int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
-  *is_zero = false;
-  /*
-  // zero'ing only happens on an initial commit... but checking upfront seems expensive..
-  _MEMORY_BASIC_INFORMATION meminfo; _mi_memzero_var(meminfo);
-  if (VirtualQuery(addr, &meminfo, size) > 0) {
-    if ((meminfo.State & MEM_COMMIT) == 0) {
-      *is_zero = true;
-    }
-  }
-  */
-  // commit
-  void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
-  if (p == NULL) return (int)GetLastError();
-  return 0;
-}
-
-int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {
-  BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT);
-  *needs_recommit = true;  // for safety, assume always decommitted even in the case of an error.
-  return (ok ? 0 : (int)GetLastError());
-}
-
-int _mi_prim_reset(void* addr, size_t size) {
-  void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
-  mi_assert_internal(p == addr);
-  #if 0
-  if (p != NULL) {
-    VirtualUnlock(addr,size); // VirtualUnlock after MEM_RESET removes the memory directly from the working set
-  }
-  #endif
-  return (p != NULL ? 0 : (int)GetLastError());
-}
-
-int _mi_prim_reuse(void* addr, size_t size) {
-  MI_UNUSED(addr); MI_UNUSED(size);
-  return 0;
-}
-
-int _mi_prim_protect(void* addr, size_t size, bool protect) {
-  DWORD oldprotect = 0;
-  BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect);
-  return (ok ? 0 : (int)GetLastError());
-}
-
-
-//---------------------------------------------
-// Huge page allocation
-//---------------------------------------------
-
-static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node)
-{
-  const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE;
-
-  win_enable_large_os_pages(NULL);
-
-  MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} };
-  // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
-  static _Atomic(size_t) mi_huge_pages_available = MI_ATOMIC_VAR_INIT(1);
-  if (pNtAllocateVirtualMemoryEx != NULL && mi_atomic_load_acquire(&mi_huge_pages_available) != 0) {
-    params[0].Type.Type = MiMemExtendedParameterAttributeFlags;
-    params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
-    ULONG param_count = 1;
-    if (numa_node >= 0) {
-      param_count++;
-      params[1].Type.Type = MiMemExtendedParameterNumaNode;
-      params[1].Arg.ULong = (unsigned)numa_node;
-    }
-    SIZE_T psize = size;
-    void* base = hint_addr;
-    LONG err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count);
-    if (err == 0 && base != NULL) {
-      return base;
-    }
-    else {
-      // fall back to regular large pages
-      mi_atomic_store_release(&mi_huge_pages_available,0); // don't try further huge pages
-      _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err);
-    }
-  }
-  // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation
-  if (pVirtualAlloc2 != NULL && numa_node >= 0) {
-    params[0].Type.Type = MiMemExtendedParameterNumaNode;
-    params[0].Arg.ULong = (unsigned)numa_node;
-    return (*pVirtualAlloc2)(GetCurrentProcess(), hint_addr, size, flags, PAGE_READWRITE, params, 1);
-  }
-
-  // otherwise use regular virtual alloc on older windows
-  return VirtualAlloc(hint_addr, size, flags, PAGE_READWRITE);
-}
-
-int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
-  *is_zero = true;
-  *addr = _mi_prim_alloc_huge_os_pagesx(hint_addr,size,numa_node);
-  return (*addr != NULL ? 0 : (int)GetLastError());
-}
-
-
-//---------------------------------------------
-// Numa nodes
-//---------------------------------------------
-
-size_t _mi_prim_numa_node(void) {
-  USHORT numa_node = 0;
-  if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) {
-    // Extended API is supported
-    MI_PROCESSOR_NUMBER pnum;
-    (*pGetCurrentProcessorNumberEx)(&pnum);
-    USHORT nnode = 0;
-    BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode);
-    if (ok) { numa_node = nnode; }
-  }
-  else if (pGetNumaProcessorNode != NULL) {
-    // Vista or earlier, use older API that is limited to 64 processors. Issue #277
-    DWORD pnum = GetCurrentProcessorNumber();
-    UCHAR nnode = 0;
-    BOOL ok = pGetNumaProcessorNode((UCHAR)pnum, &nnode);
-    if (ok) { numa_node = nnode; }
-  }
-  return numa_node;
-}
-
-size_t _mi_prim_numa_node_count(void) {
-  ULONG numa_max = 0;
-  if (pGetNumaHighestNodeNumber!=NULL) {
-    (*pGetNumaHighestNodeNumber)(&numa_max);
-  }
-  // find the highest node number that has actual processors assigned to it. Issue #282
-  while (numa_max > 0) {
-    if (pGetNumaNodeProcessorMaskEx != NULL) {
-      // Extended API is supported
-      GROUP_AFFINITY affinity;
-      if ((*pGetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) {
-        if (affinity.Mask != 0) break;  // found the maximum non-empty node
-      }
-    }
-    else {
-      // Vista or earlier, use older API that is limited to 64 processors.
-      ULONGLONG mask;
-      if (pGetNumaNodeProcessorMask != NULL) {
-        if ((*pGetNumaNodeProcessorMask)((UCHAR)numa_max, &mask)) {
-          if (mask != 0) break; // found the maximum non-empty node
-        }
-      };
-    }
-    // max node was invalid or had no processor assigned, try again
-    numa_max--;
-  }
-  return ((size_t)numa_max + 1);
-}
-
-
-//----------------------------------------------------------------
-// Clock
-//----------------------------------------------------------------
-
-static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {
-  static LARGE_INTEGER mfreq; // = 0
-  if (mfreq.QuadPart == 0LL) {
-    LARGE_INTEGER f;
-    QueryPerformanceFrequency(&f);
-    mfreq.QuadPart = f.QuadPart/1000LL;
-    if (mfreq.QuadPart == 0) mfreq.QuadPart = 1;
-  }
-  return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart);
-}
-
-mi_msecs_t _mi_prim_clock_now(void) {
-  LARGE_INTEGER t;
-  QueryPerformanceCounter(&t);
-  return mi_to_msecs(t);
-}
-
-
-//----------------------------------------------------------------
-// Process Info
-//----------------------------------------------------------------
-
-#include <psapi.h>
-
-static mi_msecs_t filetime_msecs(const FILETIME* ftime) {
-  ULARGE_INTEGER i;
-  i.LowPart = ftime->dwLowDateTime;
-  i.HighPart = ftime->dwHighDateTime;
-  mi_msecs_t msecs = (i.QuadPart / 10000); // FILETIME is in 100 nano seconds
-  return msecs;
-}
-
-typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD);
-static PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL;
-
-void _mi_prim_process_info(mi_process_info_t* pinfo)
-{
-  FILETIME ct;
-  FILETIME ut;
-  FILETIME st;
-  FILETIME et;
-  GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut);
-  pinfo->utime = filetime_msecs(&ut);
-  pinfo->stime = filetime_msecs(&st);
-
-  // load psapi on demand
-  mi_atomic_do_once {
-    HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll"));
-    if (hDll != NULL) {
-      pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo");
-      // FreeLibrary(hDll);  // don't free
-    }
-  }
-
-  // get process info
-  PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);
-  if (pGetProcessMemoryInfo != NULL) {
-    pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));
-  }
-  pinfo->current_rss    = (size_t)info.WorkingSetSize;
-  pinfo->peak_rss       = (size_t)info.PeakWorkingSetSize;
-  pinfo->current_commit = (size_t)info.PagefileUsage;
-  pinfo->peak_commit    = (size_t)info.PeakPagefileUsage;
-  pinfo->page_faults    = (size_t)info.PageFaultCount;
-}
-
-//----------------------------------------------------------------
-// Output
-//----------------------------------------------------------------
-
-void _mi_prim_out_stderr( const char* msg )
-{
-  // on windows with redirection, the C runtime cannot handle locale dependent output
-  // after the main thread closes so we use direct console output.
-  if (!_mi_preloading()) {
-    // _cputs(msg);  // _cputs cannot be used as it aborts when failing to lock the console
-    static HANDLE hcon = INVALID_HANDLE_VALUE;
-    static bool hconIsConsole = false;
-    if (hcon == INVALID_HANDLE_VALUE) {
-      hcon = GetStdHandle(STD_ERROR_HANDLE);
-      #ifdef MI_HAS_CONSOLE_IO
-      CONSOLE_SCREEN_BUFFER_INFO sbi;
-      hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi));
-      #endif
-    }
-    const size_t len = _mi_strlen(msg);
-    if (len > 0 && len < UINT32_MAX) {
-      DWORD written = 0;
-      if (hconIsConsole) {
-        #ifdef MI_HAS_CONSOLE_IO
-        WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);
-        #endif
-      }
-      else if (hcon != INVALID_HANDLE_VALUE) {
-        // use direct write if stderr was redirected
-        WriteFile(hcon, msg, (DWORD)len, &written, NULL);
-      }
-      else {
-        // finally fall back to fputs after all
-        fputs(msg, stderr);
-      }
-    }
-  }
-}
-
-
-//----------------------------------------------------------------
-// Environment
-//----------------------------------------------------------------
-
-// On Windows use GetEnvironmentVariable instead of getenv to work
-// reliably even when this is invoked before the C runtime is initialized.
-// i.e. when `_mi_preloading() == true`.
-// Note: on windows, environment names are not case sensitive.
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
-  result[0] = 0;
-  const size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size);
-  return (len > 0 && len < result_size);
-}
-
-
-//----------------------------------------------------------------
-// Random
-//----------------------------------------------------------------
-
-#if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus)
-// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using
-// dynamic overriding, we observed it can raise an exception when compiled with C++, and
-// sometimes deadlocks when also running under the VS debugger.
-// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom.
-// To be continued..
-#pragma comment (lib,"advapi32.lib")
-#define RtlGenRandom  SystemFunction036
-mi_decl_externc BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength);
-
-bool _mi_prim_random_buf(void* buf, size_t buf_len) {
-  return (RtlGenRandom(buf, (ULONG)buf_len) != 0);
-}
-
-#else
-
-#ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG
-#define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002
-#endif
-
-typedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG);
-static  PBCryptGenRandom pBCryptGenRandom = NULL;
-
-bool _mi_prim_random_buf(void* buf, size_t buf_len) {
-  mi_assert(buf_len <= ULONG_MAX);
-  if (buf_len > ULONG_MAX) return false;
-  mi_atomic_do_once {
-    HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll"));
-    if (hDll != NULL) {
-      pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom");
-      // FreeLibrary(hDll);  // don't free
-    }
-  }
-  if (pBCryptGenRandom == NULL) return false;
-  return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0);
-}
-
-#endif  // MI_USE_RTLGENRANDOM
-
-
-//----------------------------------------------------------------
-// Thread pool?
-//----------------------------------------------------------------
-
-bool _mi_prim_thread_is_in_threadpool(void) {
-#if (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64)
-  if (win_major_version >= 6) {
-    // check if this thread belongs to a windows threadpool
-    // see: <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm>
-    struct _TEB* const teb = NtCurrentTeb();
-    void* const pool_data = *((void**)((uint8_t*)teb + (MI_SIZE_BITS == 32 ? 0x0F90 : 0x1778)));
-    return (pool_data != NULL);
-  }
-#endif
-  return false;
-}
-
-void _mi_prim_thread_yield(void) {
-  SwitchToThread();
-}
-
-//----------------------------------------------------------------
-// Process & Thread Init/Done
-//----------------------------------------------------------------
-
-//static void mi_debug_out(const char* s) {
-//  HANDLE h = GetStdHandle(STD_ERROR_HANDLE);
-//  WriteConsole(h, s, (DWORD)_mi_strlen(s), NULL, NULL);
-//}
-
-static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
-  MI_UNUSED(reserved);
-  MI_UNUSED(module);
-  if (reason==DLL_PROCESS_ATTACH) {
-    _mi_auto_process_init();
-  }
-  else if (reason==DLL_PROCESS_DETACH) {
-    _mi_auto_process_done();
-  }
-  else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) {
-    _mi_thread_done(NULL);
-  }
-}
-
-
-/* -----------------------------------------------------------------------
-   Auto initialize and finalize mimalloc on process and thread start/end.
-   By default we use a combination of CRT init and TLS sections for
-   both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`).
-------------------------------------------------------------------------- */
-#if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS)
-  #if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER)
-    #define MI_WIN_INIT_USE_CRT_TLS      1  
-  #else
-    #define MI_WIN_INIT_USE_TLS_DLLMAIN  1  /* default for Intel ICX, see issue #1268 */  
-  #endif
-#endif
-
-#if defined(MI_WIN_INIT_USE_CRT_TLS)
-  #define MI_PRIM_HAS_PROCESS_ATTACH  1
-  // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
-  void _mi_prim_thread_init_auto_done(void) {}
-  void _mi_prim_thread_done_auto_done(void) {}
-  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-    MI_UNUSED(theap);
-  }
-
-  static bool mi_module_is_dll(PVOID mod) {
-    if (mod==NULL) return false;
-    PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;
-    PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);
-    return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);
-  }
-
-  static bool mi_current_module_is_dll(void) {
-    HMODULE mod = NULL;
-    const BOOL ok = GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCSTR)mi_current_module_is_dll, &mod);
-    return (ok && mi_module_is_dll(mod));
-  }
-
-  // Hook into CRT initialization and finalization.
-  static void mi_cdecl mi_crt_done(void) {
-    // mi_debug_out(mi_current_module_is_dll() ? "crt dll done\n" : "crt exe done\n");
-    mi_win_main(NULL, DLL_PROCESS_DETACH, 0);
-  }
-
-  static int mi_cdecl mi_crt_init(void) {
-    // mi_debug_out(mi_current_module_is_dll() ? "crt dll init\n" : "crt exe init\n");
-    if (mi_current_module_is_dll()) {
-      // in a dll, atexit (crt_done) is called after tls process detach
-      atexit(&mi_crt_done);
-    }
-    return 0;
-  }
-
-  // We also hook into the Windows loader TLS initialization and finalization.
-  static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {
-    if (reason == DLL_THREAD_ATTACH) {
-      //mi_debug_out("tls thread attach\n");
-      mi_win_main(module, reason, reserved);
-    }
-    else if (reason == DLL_PROCESS_ATTACH) {
-      // mi_debug_out(mi_module_is_dll(module) ? "tls dll process attach\n" : "tls exe process attach\n");
-      // tls process attach is always called before crt init
-      mi_win_main(module, reason, reserved);
-    }
-  }
-
-  static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {
-    if (reason == DLL_THREAD_DETACH) {
-      //mi_debug_out("tls thread detach\n");
-      mi_win_main(module, reason, reserved);
-    }
-    else if (reason == DLL_PROCESS_DETACH) {
-      // mi_debug_out(mi_module_is_dll(module) ? "tls dll process detach\n" : "tls exe process detach\n");
-      if (!mi_module_is_dll(module)) {
-        // in an exe, tls process detach is called after atexit (mi_crt_done)
-        mi_win_main(module, reason, reserved);
-      }
-    }
-  }
-
-  // Set up TLS callbacks in a statically linked library by using special data sections.
-  // See <https://stackoverflow.com/questions/14538159/tls-callback-in-windows>
-  // We use 2 entries to ensure we call attach events before constructors
-  // are called, and detach events after destructors are called.
-  // We also set up a CRT initialization callback.
-  #if defined(__cplusplus)
-  extern "C" {
-  #endif
-
-  typedef int (mi_cdecl* mi_crt_callback_t)(void);
-  #if defined(_WIN64)
-    #pragma comment(linker, "/INCLUDE:_tls_used")
-    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
-    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
-    #pragma const_seg(".CRT$XLB")
-      extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
-      const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
-    #pragma const_seg()
-    #pragma const_seg(".CRT$XLY")
-      extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
-      const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
-    #pragma const_seg()
-    #pragma const_seg(".CRT$XIB")
-      extern const mi_crt_callback_t _mi_crt_callback_init[];
-      const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
-    #pragma const_seg()
-  #else
-    #pragma comment(linker, "/INCLUDE:__tls_used")
-    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
-    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
-    #pragma data_seg(".CRT$XLB")
-      PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
-    #pragma data_seg()
-    #pragma data_seg(".CRT$XLY")
-      PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
-    #pragma data_seg()
-    #pragma data_seg(".CRT$XIB")
-      mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
-    #pragma data_seg()
-  #endif
-
-  #if defined(__cplusplus)
-  }
-  #endif
-
-/* ----------------------------------------------------------------
-  Here we use a combination of the raw DLL main and TLS sections for
-  both static and dynamic linkage. This is perhaps the most robust as
-  the raw DLL main ensure we can close mimalloc latest possible.
-  However, it can lead to link errors with on `_pRawDllMain` if other
-  libraries also try to hook into this (like Boost).
-*/
-#elif defined(MI_WIN_INIT_USE_RAW_DLLMAIN)
-  #define MI_PRIM_HAS_PROCESS_ATTACH  1
-  // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
-  void _mi_prim_thread_init_auto_done(void) {}
-  void _mi_prim_thread_done_auto_done(void) {}
-  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-    MI_UNUSED(theap);
-  }
-
-  // If linked into a DLL module, this raw entry is called before the CRT attach and
-  // after the CRT detach through the CRT _pRawDllMain pointer.
-  static BOOL NTAPI mi_dll_main_raw(PVOID module, DWORD reason, LPVOID reserved) {
-    mi_win_main(module, reason, reserved);
-    return TRUE;
-  }
-
-  // Set the value of the CRT _pRawDllMain pointer
-  #if defined(__cplusplus)
-  extern "C"
-  #endif
-  PVOID _pRawDllMain = (PVOID)&mi_dll_main_raw;
-
-  // We also hook into the Windows loader TLS initialization and finalization.
-  // If we are linked into an EXE module we rely on these as `mi_dll_main_raw`
-  // is not called (and otherwise we ignore the TLS callbacks by checking if we are in a DLL).
-  static bool mi_module_is_dll(PVOID mod) {
-    if (mod==NULL) return false;
-    PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;
-    PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);
-    return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);
-  }
-
-  static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {
-    if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {
-      if (!mi_module_is_dll(module)) {
-        mi_win_main(module, reason, reserved);
-      }
-    }
-  }
-
-  static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {
-    if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {
-      if (!mi_module_is_dll(module)) {
-        mi_win_main(module, reason, reserved);
-      }
-    }
-  }
-
-  // Set up TLS callbacks in a statically linked library by using special data sections.
-  // See <https://stackoverflow.com/questions/14538159/tls-callback-in-windows>
-  // We use 2 entries to ensure we call attach events before constructors
-  // are called, and detach events after destructors are called.
-  #if defined(__cplusplus)
-  extern "C" {
-  #endif
-
-  #if defined(_WIN64)
-    #pragma comment(linker, "/INCLUDE:_tls_used")
-    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
-    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
-    #pragma const_seg(".CRT$XLB")
-      extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
-      const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
-    #pragma const_seg()
-    #pragma const_seg(".CRT$XLY")
-      extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
-      const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
-    #pragma const_seg()
-  #else
-    #pragma comment(linker, "/INCLUDE:__tls_used")
-    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
-    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
-    #pragma data_seg(".CRT$XLB")
-      PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
-    #pragma data_seg()
-    #pragma data_seg(".CRT$XLY")
-      PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
-    #pragma data_seg()
-  #endif
-
-  #if defined(__cplusplus)
-  }
-  #endif
-
-/* --------------------------------------------------------------------
-   Legacy options: DllMain, TLS, and FLS
-   --------------------------------------------------------------------*/
-#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && defined(MI_SHARED_LIB)
-  #define MI_PRIM_HAS_PROCESS_ATTACH  1
-
-  // Windows DLL: easy to hook into process_init and thread_done
-  BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {
-    mi_win_main((PVOID)inst,reason,reserved);
-    return TRUE;
-  }
-
-  // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
-  void _mi_prim_thread_init_auto_done(void) { }
-  void _mi_prim_thread_done_auto_done(void) { }
-  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-    MI_UNUSED(theap);
-  }
-
-#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN)
-  #define MI_PRIM_HAS_PROCESS_ATTACH  1
-
-  static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) {
-    if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {
-      mi_win_main(module, reason, reserved);
-    }
-  }
-  static void NTAPI mi_win_main_detach(PVOID module, DWORD reason, LPVOID reserved) {
-    if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {
-      mi_win_main(module, reason, reserved);
-    }
-  }
-
-  // Set up TLS callbacks in a statically linked library by using special data sections.
-  // See <https://stackoverflow.com/questions/14538159/tls-callback-in-windows>
-  // We use 2 entries to ensure we call attach events before constructors
-  // are called, and detach events after destructors are called.
-  #if defined(__cplusplus)
-  extern "C" {
-  #endif
-
-  #if defined(_WIN64)
-    #pragma comment(linker, "/INCLUDE:_tls_used")
-    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
-    #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
-    #pragma const_seg(".CRT$XLB")
-    extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
-    const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };
-    #pragma const_seg()
-    #pragma const_seg(".CRT$XLY")
-    extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
-    const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
-    #pragma const_seg()
-  #else
-    #pragma comment(linker, "/INCLUDE:__tls_used")
-    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
-    #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
-    #pragma data_seg(".CRT$XLB")
-    PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };
-    #pragma data_seg()
-    #pragma data_seg(".CRT$XIY")
-    PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
-    #pragma data_seg()
-  #endif
-
-  #if defined(__cplusplus)
-  }
-  #endif
-
-  // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
-  void _mi_prim_thread_init_auto_done(void) { }
-  void _mi_prim_thread_done_auto_done(void) { }
-  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-    MI_UNUSED(theap);
-  }
-
-#elif defined(MI_WIN_INIT_USE_FLS) // deprecated: statically linked, use fiber api
-
-  #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`)
-    // MSVC: use data section magic for static libraries
-    // See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
-    #define MI_PRIM_HAS_PROCESS_ATTACH 1
-
-    static int mi_process_attach(void) {
-      mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL);
-      atexit(&_mi_auto_process_done);
-      return 0;
-    }
-    typedef int(*mi_crt_callback_t)(void);
-    #if defined(_WIN64)
-      #pragma comment(linker, "/INCLUDE:_mi_tls_callback")
-      #pragma section(".CRT$XIU", long, read)
-    #else
-      #pragma comment(linker, "/INCLUDE:__mi_tls_callback")
-    #endif
-    #pragma data_seg(".CRT$XIU")
-    mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach };
-    #pragma data_seg()
-  #endif
-
-  // use the fiber api for calling `_mi_thread_done`.
-  #include <fibersapi.h>
-  #if (_WIN32_WINNT < 0x600)  // before Windows Vista
-  WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );
-  WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex );
-  WINBASEAPI BOOL  WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData );
-  WINBASEAPI BOOL  WINAPI FlsFree(_In_ DWORD dwFlsIndex);
-  #endif
-
-  static DWORD mi_fls_key = (DWORD)(-1);
-
-  static void NTAPI mi_fls_done(PVOID value) {
-    mi_theap_t* theap = (mi_theap_t*)value;
-    if (theap != NULL) {
-      _mi_thread_done(theap);
-      FlsSetValue(mi_fls_key, NULL);  // prevent recursion as _mi_thread_done may set it back to the main theap, issue #672
-    }
-  }
-
-  void _mi_prim_thread_init_auto_done(void) {
-    mi_fls_key = FlsAlloc(&mi_fls_done);
-  }
-
-  void _mi_prim_thread_done_auto_done(void) {
-    // call thread-done on all threads (except the main thread) to prevent
-    // dangling callback pointer if statically linked with a DLL; Issue #208
-    FlsFree(mi_fls_key);
-  }
-
-  void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
-    mi_assert_internal(mi_fls_key != (DWORD)(-1));
-    FlsSetValue(mi_fls_key, theap);
-  }
-#else
-#error "define windows process and thread auto initialization"
-#endif
-
-// ----------------------------------------------------
-// Communicate with the redirection module on Windows
-// ----------------------------------------------------
-#if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT)
-  #define MI_PRIM_HAS_ALLOCATOR_INIT 1
-
-  static bool mi_redirected = false;   // true if malloc redirects to mi_malloc
-
-  bool _mi_is_redirected(void) {
-    return mi_redirected;
-  }
-
-  #ifdef __cplusplus
-  extern "C" {
-  #endif
-  mi_decl_export void _mi_redirect_entry(DWORD reason) {
-    // called on redirection; careful as this may be called before DllMain
-    if (reason == DLL_PROCESS_ATTACH) {
-      mi_redirected = true;
-    }
-    else if (reason == DLL_PROCESS_DETACH) {
-      mi_redirected = false;
-    }
-    else if (reason == DLL_THREAD_DETACH) {
-      // mi_debug_out("redirect thread detach\n");
-      _mi_thread_done(NULL);
-    }
-  }
-  __declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message);
-  __declspec(dllimport) void mi_cdecl mi_allocator_done(void);
-  #ifdef __cplusplus
-  }
-  #endif
-  bool _mi_allocator_init(const char** message) {
-    return mi_allocator_init(message);
-  }
-  void _mi_allocator_done(void) {
-    mi_allocator_done();
-  }
-#endif
-
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/readme.md b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/readme.md
deleted file mode 100644
index 217c3d17..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/prim/windows/readme.md
+++ /dev/null
@@ -1,17 +0,0 @@
-## Primitives:
-
-- `prim.c` contains Windows primitives for OS allocation.
-
-## Event Tracing for Windows (ETW)
-
-- `etw.h` is generated from `etw.man` which contains the manifest for mimalloc events.
-  (100 is an allocation, 101 is for a free)
-
-- `etw-mimalloc.wprp` is a profile for the Windows Performance Recorder (WPR).
-  In an admin prompt, you can use:
-  ```
-  > wpr -start src\prim\windows\etw-mimalloc.wprp -filemode
-  > <my mimalloc program>
-  > wpr -stop test.etl
-  ``` 
-  and then open `test.etl` in the Windows Performance Analyzer (WPA).
\ No newline at end of file
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/random.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/random.c
deleted file mode 100644
index 75b5cfbb..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/random.c
+++ /dev/null
@@ -1,258 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2021, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"    // _mi_prim_random_buf
-
-/* ----------------------------------------------------------------------------
-We use our own PRNG to keep predictable performance of random number generation
-and to avoid implementations that use a lock. We only use the OS provided
-random source to initialize the initial seeds. Since we do not need ultimate
-performance but we do rely on the security (for secret cookies in secure mode)
-we use a cryptographically secure generator (chacha20).
------------------------------------------------------------------------------*/
-
-#define MI_CHACHA_ROUNDS (20)   // perhaps use 12 for better performance?
-
-
-/* ----------------------------------------------------------------------------
-Chacha20 implementation as the original algorithm with a 64-bit nonce
-and counter: https://en.wikipedia.org/wiki/Salsa20
-The input matrix has sixteen 32-bit values:
-Position  0 to  3: constant key
-Position  4 to 11: the key
-Position 12 to 13: the counter.
-Position 14 to 15: the nonce.
-
-The implementation uses regular C code which compiles very well on modern compilers.
-(gcc x64 has no register spills, and clang 6+ uses SSE instructions)
------------------------------------------------------------------------------*/
-
-static inline void qround(uint32_t x[16], size_t a, size_t b, size_t c, size_t d) {
-  x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 16);
-  x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 12);
-  x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 8);
-  x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 7);
-}
-
-static void chacha_block(mi_random_ctx_t* ctx)
-{
-  // scramble into `x`
-  uint32_t x[16];
-  for (size_t i = 0; i < 16; i++) {
-    x[i] = ctx->input[i];
-  }
-  for (size_t i = 0; i < MI_CHACHA_ROUNDS; i += 2) {
-    qround(x, 0, 4,  8, 12);
-    qround(x, 1, 5,  9, 13);
-    qround(x, 2, 6, 10, 14);
-    qround(x, 3, 7, 11, 15);
-    qround(x, 0, 5, 10, 15);
-    qround(x, 1, 6, 11, 12);
-    qround(x, 2, 7,  8, 13);
-    qround(x, 3, 4,  9, 14);
-  }
-
-  // add scrambled data to the initial state
-  for (size_t i = 0; i < 16; i++) {
-    ctx->output[i] = x[i] + ctx->input[i];
-  }
-  ctx->output_available = 16;
-
-  // increment the counter for the next round
-  ctx->input[12] += 1;
-  if (ctx->input[12] == 0) {
-    ctx->input[13] += 1;
-    if (ctx->input[13] == 0) {  // and keep increasing into the nonce
-      ctx->input[14] += 1;
-    }
-  }
-}
-
-static uint32_t chacha_next32(mi_random_ctx_t* ctx) {
-  if (ctx->output_available <= 0) {
-    chacha_block(ctx);
-    ctx->output_available = 16; // (assign again to suppress static analysis warning)
-  }
-  const uint32_t x = ctx->output[16 - ctx->output_available];
-  ctx->output[16 - ctx->output_available] = 0; // reset once the data is handed out
-  ctx->output_available--;
-  return x;
-}
-
-static inline uint32_t read32(const uint8_t* p, size_t idx32) {
-  const size_t i = 4*idx32;
-  return ((uint32_t)p[i+0] | (uint32_t)p[i+1] << 8 | (uint32_t)p[i+2] << 16 | (uint32_t)p[i+3] << 24);
-}
-
-static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t nonce)
-{
-  // since we only use chacha for randomness (and not encryption) we
-  // do not _need_ to read 32-bit values as little endian but we do anyways
-  // just for being compatible :-)
-  ctx->output_available = 0;
-  _mi_memzero(ctx->output,sizeof(ctx->output));
-  for (size_t i = 0; i < 4; i++) {
-    const uint8_t* sigma = (uint8_t*)"expand 32-byte k";
-    ctx->input[i] = read32(sigma,i);
-  }
-  for (size_t i = 0; i < 8; i++) {
-    ctx->input[i + 4] = read32(key,i);
-  }
-  ctx->input[12] = 0;
-  ctx->input[13] = 0;
-  ctx->input[14] = (uint32_t)nonce;
-  ctx->input[15] = (uint32_t)(nonce >> 32);
-}
-
-static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) {
-  _mi_memzero(ctx_new, sizeof(*ctx_new));
-  ctx_new->weak = ctx->weak;
-  _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input));
-  ctx_new->input[12] = 0;
-  ctx_new->input[13] = 0;
-  ctx_new->input[14] = (uint32_t)nonce;
-  ctx_new->input[15] = (uint32_t)(nonce >> 32);
-  mi_assert_internal(ctx->input[14] != ctx_new->input[14] || ctx->input[15] != ctx_new->input[15]); // do not reuse nonces!
-  chacha_block(ctx_new);
-}
-
-
-/* ----------------------------------------------------------------------------
-Random interface
------------------------------------------------------------------------------*/
-
-#if MI_DEBUG>1
-static bool mi_random_is_initialized(mi_random_ctx_t* ctx) {
-  return (ctx != NULL && ctx->input[0] != 0);
-}
-#endif
-
-void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) {
-  mi_assert_internal(mi_random_is_initialized(ctx));
-  mi_assert_internal(ctx != ctx_new);
-  chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new);
-}
-
-uintptr_t _mi_random_next(mi_random_ctx_t* ctx) {
-  mi_assert_internal(mi_random_is_initialized(ctx));
-  uintptr_t r;
-  do {
-    #if MI_INTPTR_SIZE <= 4
-    r = chacha_next32(ctx);
-    #elif MI_INTPTR_SIZE == 8
-    r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));
-    #else
-    # error "define mi_random_next for this platform"
-    #endif
-  } while (r==0);
-  return r;
-}
-
-
-/* ----------------------------------------------------------------------------
-To initialize a fresh random context.
-If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR.
------------------------------------------------------------------------------*/
-
-uintptr_t _mi_os_random_weak(uintptr_t extra_seed) {
-  uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random
-  x ^= _mi_prim_clock_now();
-  // and do a few randomization steps
-  uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1;
-  for (uintptr_t i = 0; i < max || x==0; i++, x++) {
-    x = _mi_random_shuffle(x);
-  }
-  mi_assert_internal(x != 0);
-  return x;
-}
-
-static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {
-  uint8_t key[32];
-  if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) {
-    // if we fail to get random data from the OS, we fall back to a
-    // weak random source based on the current time
-    #if !defined(__wasi__)
-    if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); }
-    #endif
-    uintptr_t x = _mi_os_random_weak(0);
-    for (size_t i = 0; i < 32; i+=4, x++) {  
-      x = _mi_random_shuffle(x);
-      key[i]   = (uint8_t)(x);
-      key[i+1] = (uint8_t)(x>>8);
-      key[i+2] = (uint8_t)(x>>16);
-      key[i+3] = (uint8_t)(x>>24);      
-    }
-    ctx->weak = true;
-  }
-  else {
-    ctx->weak = false;
-  }
-  chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ );
-}
-
-void _mi_random_init(mi_random_ctx_t* ctx) {
-  mi_random_init_ex(ctx, false);
-}
-
-void _mi_random_init_weak(mi_random_ctx_t * ctx) {
-  mi_random_init_ex(ctx, true);
-}
-
-void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx) {
-  if (ctx->weak) {
-    _mi_random_init(ctx);
-  }
-}
-
-/* --------------------------------------------------------
-test vectors from <https://tools.ietf.org/html/rfc8439>
------------------------------------------------------------ */
-/*
-static bool array_equals(uint32_t* x, uint32_t* y, size_t n) {
-  for (size_t i = 0; i < n; i++) {
-    if (x[i] != y[i]) return false;
-  }
-  return true;
-}
-static void chacha_test(void)
-{
-  uint32_t x[4] = { 0x11111111, 0x01020304, 0x9b8d6f43, 0x01234567 };
-  uint32_t x_out[4] = { 0xea2a92f4, 0xcb1cf8ce, 0x4581472e, 0x5881c4bb };
-  qround(x, 0, 1, 2, 3);
-  mi_assert_internal(array_equals(x, x_out, 4));
-
-  uint32_t y[16] = {
-       0x879531e0,  0xc5ecf37d,  0x516461b1,  0xc9a62f8a,
-       0x44c20ef3,  0x3390af7f,  0xd9fc690b,  0x2a5f714c,
-       0x53372767,  0xb00a5631,  0x974c541a,  0x359e9963,
-       0x5c971061,  0x3d631689,  0x2098d9d6,  0x91dbd320 };
-  uint32_t y_out[16] = {
-       0x879531e0,  0xc5ecf37d,  0xbdb886dc,  0xc9a62f8a,
-       0x44c20ef3,  0x3390af7f,  0xd9fc690b,  0xcfacafd2,
-       0xe46bea80,  0xb00a5631,  0x974c541a,  0x359e9963,
-       0x5c971061,  0xccc07c79,  0x2098d9d6,  0x91dbd320 };
-  qround(y, 2, 7, 8, 13);
-  mi_assert_internal(array_equals(y, y_out, 16));
-
-  mi_random_ctx_t r = {
-    { 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574,
-      0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,
-      0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c,
-      0x00000001, 0x09000000, 0x4a000000, 0x00000000 },
-    {0},
-    0
-  };
-  uint32_t r_out[16] = {
-       0xe4e7f110, 0x15593bd1, 0x1fdd0f50, 0xc47120a3,
-       0xc7f4d1c7, 0x0368c033, 0x9aaa2204, 0x4e6cd4c3,
-       0x466482d2, 0x09aa9f07, 0x05d7c214, 0xa2028bd9,
-       0xd19c12b5, 0xb94e16de, 0xe883d0cb, 0x4e3c50a2 };
-  chacha_block(&r);
-  mi_assert_internal(array_equals(r.output, r_out, 16));
-}
-*/
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/static.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/static.c
deleted file mode 100644
index 2383f659..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/static.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#ifndef _DEFAULT_SOURCE
-#define _DEFAULT_SOURCE
-#endif
-#if defined(__sun)
-// same remarks as os.c for the static's context.
-#undef _XOPEN_SOURCE
-#undef _POSIX_C_SOURCE
-#endif
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-
-// For a static override we create a single object file
-// containing the whole library. If it is linked first
-// it will override all the standard library allocation
-// functions (on Unix's).
-#include "alloc.c"          // includes alloc-override.c and free.c
-#include "alloc-aligned.c"
-#include "alloc-posix.c"
-#include "arena.c"
-#include "arena-meta.c"
-#include "bitmap.c"
-#include "heap.c"
-#include "init.c"
-#include "libc.c"
-#include "options.c"
-#include "os.c"
-#include "page.c"           // includes page-queue.c
-#include "page-map.c"
-#include "random.c"
-#include "stats.c"
-#include "theap.c"
-#include "threadlocal.c"
-#include "prim/prim.c"
-#if MI_OSX_ZONE
-#include "prim/osx/alloc-override-zone.c"
-#endif
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/stats.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/stats.c
deleted file mode 100644
index 7ca18e23..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/stats.c
+++ /dev/null
@@ -1,813 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#include "mimalloc.h"
-#include "mimalloc-stats.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/atomic.h"
-#include "mimalloc/prim.h"
-
-#include <string.h> // memset
-
-#if defined(_MSC_VER) && (_MSC_VER < 1920)
-#pragma warning(disable:4204)  // non-constant aggregate initializer
-#endif
-
-/* -----------------------------------------------------------
-  Statistics operations
------------------------------------------------------------ */
-
-static void mi_stat_update_mt(mi_stat_count_t* stat, int64_t amount) {
-  if (amount == 0) return;
-  // add atomically
-  int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount);
-  mi_atomic_maxi64_relaxed(&stat->peak, current + amount);
-  if (amount > 0) {
-    mi_atomic_addi64_relaxed(&stat->total, amount);
-  }
-}
-
-static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
-  if (amount == 0) return;
-  // add thread local
-  stat->current += amount;
-  if (stat->current > stat->peak) { stat->peak = stat->current; }
-  if (amount > 0) { stat->total += amount; }
-}
-
-
-void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) {
-  mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount);
-}
-
-void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
-  stat->total += amount;
-}
-
-void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) {
-  mi_stat_update_mt(stat, (int64_t)amount);
-}
-void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) {
-  mi_stat_update(stat, (int64_t)amount);
-}
-
-void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) {
-  mi_stat_update_mt(stat, -((int64_t)amount));
-}
-void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {
-  mi_stat_update(stat, -((int64_t)amount));
-}
-
-
-// Adjust stats to compensate; for example before committing a range,
-// first adjust downwards with parts that were already committed so
-// we avoid double counting.
-static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) {
-  if (amount == 0) return;
-  // adjust atomically
-  mi_atomic_addi64_relaxed(&stat->current, amount);
-  mi_atomic_addi64_relaxed(&stat->total, amount);
-}
-
-static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) {
-  if (amount == 0) return;
-  stat->current += amount;
-  stat->total += amount;
-}
-
-void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) {
-  mi_stat_adjust_mt(stat, (int64_t)amount);
-}
-void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) {
-  mi_stat_adjust(stat, (int64_t)amount);
-}
-void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) {
-  mi_stat_adjust_mt(stat, -((int64_t)amount));
-}
-void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {
-  mi_stat_adjust(stat, -((int64_t)amount));
-}
-
-
-// must be thread safe as it is called from stats_merge
-static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) {
-  if (stat==src) return;
-  mi_atomic_void_addi64_relaxed(&stat->total, &src->total);
-  const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current);
-
-  // Global current plus thread peak approximates new global peak
-  // note: peak scores do really not work across threads.
-  // we used to just add them together but that often overestimates in practice.
-  // similarly, max does not seem to work well. The current approach
-  // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112
-  // for a longer description.
-  mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak);
-}
-
-static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) {
-  if (stat==src) return;
-  mi_atomic_void_addi64_relaxed(&stat->total, &src->total);
-}
-
-#define MI_STAT_COUNT(stat)    mi_stat_count_add_mt(&stats->stat, &src->stat);
-#define MI_STAT_COUNTER(stat)  mi_stat_counter_add_mt(&stats->stat, &src->stat);
-
-// must be thread safe as it is called from stats_merge
-static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
-  if (stats==NULL || src==NULL || stats==src) return;
-
-  // copy all fields
-  MI_STAT_FIELDS()
-
-  #if MI_STAT>1
-  for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
-    mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]);
-  }
-  #endif
-  for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
-    mi_stat_count_add_mt(&stats->page_bins[i], &src->page_bins[i]);
-  }
-}
-
-#undef MI_STAT_COUNT
-#undef MI_STAT_COUNTER
-
-/* -----------------------------------------------------------
-  Display statistics
------------------------------------------------------------ */
-
-// unit > 0 : size in binary bytes
-// unit == 0: count as decimal
-// unit < 0 : count in binary
-static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) {
-  char buf[32]; _mi_memzero_var(buf);
-  int  len = 32;
-  const char* suffix = (unit <= 0 ? " " : "B");
-  const int64_t base = (unit == 0 ? 1000 : 1024);
-  if (unit>0) n *= unit;
-
-  const int64_t pos = (n < 0 ? -n : n);
-  if (pos < base) {
-    if (n!=1 || suffix[0] != 'B') {  // skip printing 1 B for the unit column
-      _mi_snprintf(buf, len, "%lld   %-3s", (long long)n, (n==0 ? "" : suffix));
-    }
-  }
-  else {
-    int64_t divider = base;
-    const char* magnitude = "K";
-    if (pos >= divider*base) { divider *= base; magnitude = "M"; }
-    if (pos >= divider*base) { divider *= base; magnitude = "G"; }
-    const int64_t tens = (n / (divider/10));
-    const long whole = (long)(tens/10);
-    const long frac1 = (long)(tens%10);
-    char unitdesc[8];
-    _mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix);
-    _mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);
-  }
-  _mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf);
-}
-
-
-static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
-  mi_printf_amount(n,unit,out,arg,NULL);
-}
-
-static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
-  if (unit==1) _mi_fprintf(out, arg, "%12s"," ");
-          else mi_print_amount(n,0,out,arg);
-}
-
-static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) {
-  _mi_fprintf(out, arg,"  %-10s:", msg);
-  if (unit != 0) {
-    if (unit > 0) {
-      mi_print_amount(stat->peak, unit, out, arg);
-      mi_print_amount(stat->total, unit, out, arg);
-      // mi_print_amount(stat->freed, unit, out, arg);
-      mi_print_amount(stat->current, unit, out, arg);
-      mi_print_amount(unit, 1, out, arg);
-      mi_print_count(stat->total, unit, out, arg);
-    }
-    else {
-      mi_print_amount(stat->peak, -1, out, arg);
-      mi_print_amount(stat->total, -1, out, arg);
-      // mi_print_amount(stat->freed, -1, out, arg);
-      mi_print_amount(stat->current, -1, out, arg);
-      if (unit == -1) {
-        _mi_fprintf(out, arg, "%24s", "");
-      }
-      else {
-        mi_print_amount(-unit, 1, out, arg);
-        mi_print_count((stat->total / -unit), 0, out, arg);
-      }
-    }
-    if (stat->current != 0) {
-      _mi_fprintf(out, arg, "  ");
-      _mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok));
-      _mi_fprintf(out, arg, "\n");
-    }
-    else {
-      _mi_fprintf(out, arg, "  ok\n");
-    }
-  }
-  else {
-    mi_print_amount(stat->peak, 0, out, arg);
-    mi_print_amount(stat->total, 0, out, arg);
-    mi_print_amount(stat->current, 0, out, arg);
-    _mi_fprintf(out, arg, "\n");
-  }
-}
-
-static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
-  mi_stat_print_ex(stat, msg, unit, out, arg, NULL);
-}
-
-#if MI_STAT>1
-static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
-  _mi_fprintf(out, arg, "  %-10s:", msg);
-  _mi_fprintf(out, arg, "%12s", " ");  // no peak
-  mi_print_amount(stat->total, unit, out, arg);
-  _mi_fprintf(out, arg, "\n");
-}
-#endif
-
-static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
-  _mi_fprintf(out, arg, "  %-10s:", msg);
-  mi_print_amount(stat->total, 0, out, arg);
-  _mi_fprintf(out, arg, "\n");
-}
-
-static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
-  _mi_fprintf(out, arg, "  %-10s:", msg);
-  mi_print_amount(stat->total, 1, out, arg);
-  _mi_fprintf(out, arg, "\n");
-}
-
-static void mi_stat_average_print(int64_t count, int64_t total, const char* msg, mi_output_fun* out, void* arg) {
-  const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count));
-  const int64_t avg_whole = avg_tens/10;
-  const int64_t avg_frac1 = avg_tens%10;
-  _mi_fprintf(out, arg, "  %-10s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1);
-}
-
-
-static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) {
-  _mi_fprintf(out, arg, " %-11s %11s %11s %11s %11s %11s\n",
-                        name, "peak   ", "total   ", "current   ", "block   ", "total#   ");
-}
-
-#if MI_STAT>1
-static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) {
-  bool found = false;
-  char buf[64];
-  for (size_t i = 0; i <= max; i++) {
-    if (bins[i].total > 0) {
-      found = true;
-      const size_t unit = _mi_bin_size((uint8_t)i);
-      const char* pagekind = (unit <= MI_SMALL_MAX_OBJ_SIZE ? "S" :
-                               (unit <= MI_MEDIUM_MAX_OBJ_SIZE ? "M" :
-                                 (unit <= MI_LARGE_MAX_OBJ_SIZE ? "L" : "H")));
-      _mi_snprintf(buf, 64, "bin%2s  %3lu", pagekind, (long)i);
-      mi_stat_print(&bins[i], buf, (int64_t)unit, out, arg);
-    }
-  }
-  if (found) {
-    _mi_fprintf(out, arg, "\n");
-  }
-  return found;
-}
-#endif
-
-
-
-//------------------------------------------------------------
-// Use an output wrapper for line-buffered output
-// (which is nice when using loggers etc.)
-//------------------------------------------------------------
-typedef struct buffered_s {
-  mi_output_fun* out;   // original output function
-  void*          arg;   // and state
-  char*          buf;   // local buffer of at least size `count+1`
-  size_t         used;  // currently used chars `used <= count`
-  size_t         count; // total chars available for output
-} buffered_t;
-
-static void mi_buffered_flush(buffered_t* buf) {
-  buf->buf[buf->used] = 0;
-  _mi_fputs(buf->out, buf->arg, NULL, buf->buf);
-  buf->used = 0;
-}
-
-static void mi_cdecl mi_buffered_out(const char* msg, void* arg) {
-  buffered_t* buf = (buffered_t*)arg;
-  if (msg==NULL || buf==NULL) return;
-  for (const char* src = msg; *src != 0; src++) {
-    char c = *src;
-    if (buf->used >= buf->count) mi_buffered_flush(buf);
-    mi_assert_internal(buf->used < buf->count);
-    buf->buf[buf->used++] = c;
-    if (c == '\n') mi_buffered_flush(buf);
-  }
-}
-
-//------------------------------------------------------------
-// Print statistics
-//------------------------------------------------------------
-
-mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept
-{
-  size_t elapsed;
-  size_t user_time;
-  size_t sys_time;
-  size_t current_rss;
-  size_t peak_rss;
-  size_t current_commit;
-  size_t peak_commit;
-  size_t page_faults;
-  mi_process_info(&elapsed, &user_time, &sys_time, &current_rss, &peak_rss, &current_commit, &peak_commit, &page_faults);
-  _mi_fprintf(out, arg, "  %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000);
-  _mi_fprintf(out, arg, "  %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process",
-    user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults);
-  mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s");
-  if (peak_commit > 0) {
-    _mi_fprintf(out, arg, ", peak commit: ");
-    mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s");
-  }
-  _mi_fprintf(out, arg, "\n");
-}
-
-void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept {
-  // wrap the output function to be line buffered
-  char buf[256]; _mi_memzero_var(buf);
-  buffered_t buffer = { out0, arg0, NULL, 0, 255 };
-  buffer.buf = buf;
-  mi_output_fun* out = &mi_buffered_out;
-  void* arg = &buffer;
-
-  // and print using that
-  _mi_fprintf(out, arg, "%s %zu\n", name, id);
-
-  if (stats->malloc_normal.total + stats->malloc_huge.total != 0) {
-    #if MI_STAT>1
-    mi_print_header("blocks", out, arg);
-    mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg);
-    #endif
-    #if MI_STAT
-    mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg);
-    mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg);
-    mi_stat_count_t total = { 0,0,0 };
-    mi_stat_count_add_mt(&total, &stats->malloc_normal);
-    mi_stat_count_add_mt(&total, &stats->malloc_huge);
-    mi_stat_print_ex(&total, "total", 1, out, arg, "");
-    #if MI_STAT>1
-    mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg);
-    #endif
-    _mi_fprintf(out, arg, "\n");
-    #endif
-  }
-
-  if (stats->pages.total != 0) {
-    mi_print_header("pages", out, arg);
-    mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, "");
-    // mi_stat_print(&stats->segments, "segments", -1, out, arg);
-    // mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg);
-    // mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg);
-    mi_stat_print(&stats->pages, "pages", 0, out, arg);
-    mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg);
-    mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg);
-    mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg);
-    mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg);
-    mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg);
-    mi_stat_counter_print(&stats->pages_extended, "extended", out, arg);
-    mi_stat_counter_print(&stats->pages_retire, "retire", out, arg);
-    mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg);
-    _mi_fprintf(out, arg, "\n");
-  }
-
-  if (stats->arena_count.total > 0) {
-    mi_print_header("arenas", out, arg);
-    mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, "");
-    mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, "");
-    mi_stat_counter_print_size(&stats->reset, "reset", out, arg);
-    mi_stat_counter_print_size(&stats->purged, "purged", out, arg);
-
-    mi_stat_counter_print(&stats->arena_count, "arenas", out, arg);
-    mi_stat_counter_print(&stats->arena_rollback_count, "rollback", out, arg);
-    mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg);
-    mi_stat_counter_print(&stats->commit_calls, "commits", out, arg);
-    mi_stat_counter_print(&stats->reset_calls, "resets", out, arg);
-    mi_stat_counter_print(&stats->purge_calls, "purges", out, arg);
-    mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg);
-    mi_stat_print_ex(&stats->theaps, "theaps", 0, out, arg, "");
-    mi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, "");
-    mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg);
-    _mi_fprintf(out, arg, "\n");
-
-    mi_print_header("process", out, arg);
-    mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, "");
-    _mi_fprintf(out, arg, "  %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count());
-    mi_process_info_print_out(out, arg);
-  }
-  _mi_fprintf(out, arg, "\n");
-}
-
-
-static mi_msecs_t mi_process_start; // = 0
-
-// called on process init
-void _mi_stats_init(void) {
-  if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); };
-}
-
-static void mi_stats_add_into(mi_stats_t* to, const mi_stats_t* from) {
-  mi_assert_internal(to != NULL && from != NULL);
-  if (to == from) return;
-  mi_stats_add(to, from);
-}
-
-void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) {
-  mi_assert_internal(to != NULL && from != NULL);
-  if (to == from) return;
-  mi_stats_add(to, from);
-  _mi_memzero(from, sizeof(mi_stats_t));
-}
-
-static const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept {
-  mi_stats_t* stats = &theap->stats;
-  mi_stats_t* heap_stats = &_mi_theap_heap(theap)->stats;
-  _mi_stats_merge_into( heap_stats, stats );
-  return heap_stats;
-}
-
-static const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) {
-  if (heap==NULL) { heap = mi_heap_main(); }
-  mi_theap_t* theap = _mi_heap_theap_peek(heap);
-  if (theap==NULL) return &heap->stats;
-              else return mi_stats_merge_theap_to_heap(theap);
-}
-
-// deprecated
-void mi_stats_reset(void) mi_attr_noexcept {
-  if (!mi_theap_is_initialized(_mi_theap_default())) return;
-  mi_heap_get_stats(mi_heap_main());
-  mi_heap_stats_merge_to_subproc(mi_heap_main());
-}
-
-
-void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept {
-  if (heap==NULL) { heap = mi_heap_main(); }
-  _mi_stats_print("heap", heap->heap_seq, mi_heap_get_stats(heap), out, arg);
-}
-
-typedef struct mi_heap_print_visit_info_s {
-  mi_output_fun* out;
-  void* out_arg;
-} mi_heap_print_visit_info_t;
-
-static bool mi_cdecl mi_heap_print_visitor(mi_heap_t* heap, void* arg) {
-  mi_heap_print_visit_info_t* vinfo = (mi_heap_print_visit_info_t*)(arg);
-  mi_heap_stats_print_out(heap, vinfo->out, vinfo->out_arg);
-  return true;
-}
-
-
-// show each heap and then the subproc
-void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept {
-  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
-  if (subproc==NULL) return;
-  mi_heap_print_visit_info_t vinfo = { out, arg };
-  mi_subproc_visit_heaps(subproc_id, &mi_heap_print_visitor, &vinfo);
-  _mi_stats_print("subproc", subproc->subproc_seq, &subproc->stats, out, arg);
-}
-
-
-// aggregate all stats from the heaps and subproc and print those
-void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept {
-  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
-  if (subproc==NULL) return;
-  mi_stats_t_decl(stats);
-  if (mi_subproc_stats_get(subproc_id, &stats)) {
-    _mi_stats_print("subproc", subproc->subproc_seq, &stats, out, arg);
-  }
-}
-
-void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
-  mi_subproc_stats_print_out(mi_subproc_current(),out, arg);
-}
-
-// deprecated
-void mi_stats_print(void* out) mi_attr_noexcept {
-  // for compatibility there is an `out` parameter (which can be `stdout` or `stderr`)
-  mi_stats_print_out((mi_output_fun*)out, NULL);
-}
-
-// deprecated
-void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
-  mi_theap_t* theap = _mi_theap_default();
-  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
-  _mi_stats_print("heap", _mi_theap_heap(theap)->heap_seq, &theap->stats, out, arg);
-  mi_stats_merge_theap_to_heap(_mi_theap_default());
-}
-
-
-// ----------------------------------------------------------------
-// Basic timer for convenience; use milli-seconds to avoid doubles
-// ----------------------------------------------------------------
-
-static mi_msecs_t mi_clock_diff;
-
-mi_msecs_t _mi_clock_now(void) {
-  return _mi_prim_clock_now();
-}
-
-mi_msecs_t _mi_clock_start(void) {
-  if (mi_clock_diff == 0.0) {
-    mi_msecs_t t0 = _mi_clock_now();
-    mi_clock_diff = _mi_clock_now() - t0;
-  }
-  return _mi_clock_now();
-}
-
-mi_msecs_t _mi_clock_end(mi_msecs_t start) {
-  mi_msecs_t end = _mi_clock_now();
-  return (end - start - mi_clock_diff);
-}
-
-
-// --------------------------------------------------------
-// Basic process statistics
-// --------------------------------------------------------
-
-mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept
-{
-  mi_process_info_t pinfo;
-  _mi_memzero_var(pinfo);
-  pinfo.elapsed        = _mi_clock_end(mi_process_start);
-  { const mi_subproc_t* subproc = _mi_subproc_main();
-    if (subproc!=NULL) {
-      pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)));
-      pinfo.peak_commit    = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)));
-    }  
-  }
-  pinfo.current_rss    = pinfo.current_commit;
-  pinfo.peak_rss       = pinfo.peak_commit;
-  pinfo.utime          = 0;
-  pinfo.stime          = 0;
-  pinfo.page_faults    = 0;
-
-  _mi_prim_process_info(&pinfo);
-
-  if (elapsed_msecs!=NULL)  *elapsed_msecs  = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX));
-  if (user_msecs!=NULL)     *user_msecs     = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX));
-  if (system_msecs!=NULL)   *system_msecs   = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX));
-  if (current_rss!=NULL)    *current_rss    = pinfo.current_rss;
-  if (peak_rss!=NULL)       *peak_rss       = pinfo.peak_rss;
-  if (current_commit!=NULL) *current_commit = pinfo.current_commit;
-  if (peak_commit!=NULL)    *peak_commit    = pinfo.peak_commit;
-  if (page_faults!=NULL)    *page_faults    = pinfo.page_faults;
-}
-
-mi_decl_export void mi_process_info_print(void) mi_attr_noexcept {
-  mi_process_info_print_out(NULL, NULL);
-}
-
-
-// --------------------------------------------------------
-// Return statistics
-// --------------------------------------------------------
-
-size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept {
-  if (bin > MI_BIN_HUGE) return 0;
-  return _mi_bin_size(bin);
-}
-
-static bool mi_stats_copy(mi_stats_t* stats_to, const mi_stats_t* stats_from) mi_attr_noexcept {
-  if (stats_to == NULL || stats_to->size != sizeof(mi_stats_t) || stats_to->version != MI_STAT_VERSION) return false;
-  if (stats_from == NULL || stats_from->size != stats_to->size) return false;
-  _mi_memcpy(stats_to, stats_from, stats_to->size);
-  return true;
-}
-
-bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {
-  const mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
-  if (subproc==NULL) return false;
-  return mi_stats_copy(stats,&subproc->stats);
-}
-
-bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept {
-  return mi_stats_copy(stats, mi_heap_get_stats(heap));
-}
-
-
-static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) {
-  mi_stats_t* stats = (mi_stats_t*)arg;
-  mi_stats_add_into(stats, mi_heap_get_stats(heap));
-  return true;
-}
-
-bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {
-  if (stats==NULL) return false;
-  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
-  if (subproc == NULL) return false;
-  if (!mi_stats_copy(stats, &subproc->stats)) return false;
-  mi_subproc_visit_heaps(subproc_id, &mi_heap_aggregate_visitor, stats);
-  return true;
-}
-
-bool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept {
-  return mi_subproc_stats_get(mi_subproc_current(), stats);
-}
-
-
-// --------------------------------------------------------
-// Statics in json format
-// --------------------------------------------------------
-
-typedef struct mi_json_buf_s {
-  char*   buf;
-  size_t  size;
-  size_t  used;
-  bool    can_realloc;
-} mi_json_buf_t;
-
-static bool mi_json_buf_expand(mi_json_buf_t* hbuf) {
-  if (hbuf==NULL) return false;
-  if (hbuf->buf != NULL && hbuf->size>0) {
-    hbuf->buf[hbuf->size-1] = 0;
-  }
-  if (hbuf->size > SIZE_MAX/2 || !hbuf->can_realloc) return false;
-  const size_t newsize = (hbuf->size == 0 ? mi_good_size(12*MI_KiB) : 2*hbuf->size);
-  char* const  newbuf  = (char*)mi_rezalloc(hbuf->buf, newsize);
-  if (newbuf == NULL) return false;
-  hbuf->buf = newbuf;
-  hbuf->size = newsize;
-  return true;
-}
-
-static void mi_json_buf_print(mi_json_buf_t* hbuf, const char* msg) {
-  if (msg==NULL || hbuf==NULL) return;
-  if (hbuf->used + 1 >= hbuf->size && !hbuf->can_realloc) return;
-  for (const char* src = msg; *src != 0; src++) {
-    char c = *src;
-    if (hbuf->used + 1 >= hbuf->size) {
-      if (!mi_json_buf_expand(hbuf)) return;
-    }
-    mi_assert_internal(hbuf->used < hbuf->size);
-    hbuf->buf[hbuf->used++] = c;
-  }
-  mi_assert_internal(hbuf->used < hbuf->size);
-  hbuf->buf[hbuf->used] = 0;
-}
-
-static void mi_json_buf_print_count_bin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, size_t bin, bool add_comma) {
-  const size_t binsize = mi_stats_get_bin_size(bin);
-  const size_t pagesize = (binsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_SMALL_PAGE_SIZE :
-                            (binsize <= MI_MEDIUM_MAX_OBJ_SIZE ? MI_MEDIUM_PAGE_SIZE :
-                              (binsize <= MI_LARGE_MAX_OBJ_SIZE ? MI_LARGE_PAGE_SIZE : 0)));
-  char buf[128];
-  _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"block_size\": %zu, \"page_size\": %zu }%s\n", prefix, stat->total, stat->peak, stat->current, binsize, pagesize, (add_comma ? "," : ""));
-  buf[127] = 0;
-  mi_json_buf_print(hbuf, buf);
-}
-
-static void mi_json_buf_print_count_cbin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, mi_chunkbin_t bin, bool add_comma) {
-  const char* cbin = " ";
-  switch(bin) {
-    case MI_CBIN_SMALL:  cbin = "S"; break;
-    case MI_CBIN_MEDIUM: cbin = "M"; break;
-    case MI_CBIN_LARGE:  cbin = "L"; break;
-    case MI_CBIN_HUGE:   cbin = "H"; break;
-    case MI_CBIN_OTHER:  cbin = "X"; break;
-    default: cbin = " "; break;
-  }
-  char buf[128];
-  _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"bin\": \"%s\" }%s\n", prefix, stat->total, stat->peak, stat->current, cbin, (add_comma ? "," : ""));
-  buf[127] = 0;
-  mi_json_buf_print(hbuf, buf);
-}
-
-static void mi_json_buf_print_count(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, bool add_comma) {
-  char buf[128];
-  _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld }%s\n", prefix, stat->total, stat->peak, stat->current, (add_comma ? "," : ""));
-  buf[127] = 0;
-  mi_json_buf_print(hbuf, buf);
-}
-
-static void mi_json_buf_print_count_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_count_t* stat) {
-  char buf[128];
-  _mi_snprintf(buf, 128, "  \"%s\": ", name);
-  buf[127] = 0;
-  mi_json_buf_print(hbuf, buf);
-  mi_json_buf_print_count(hbuf, "", stat, true);
-}
-
-static void mi_json_buf_print_value(mi_json_buf_t* hbuf, const char* name, int64_t val) {
-  char buf[128];
-  _mi_snprintf(buf, 128, "  \"%s\": %lld,\n", name, val);
-  buf[127] = 0;
-  mi_json_buf_print(hbuf, buf);
-}
-
-static void mi_json_buf_print_size(mi_json_buf_t* hbuf, const char* name, size_t val, bool add_comma) {
-  char buf[128];
-  _mi_snprintf(buf, 128, "    \"%s\": %zu%s\n", name, val, (add_comma ? "," : ""));
-  buf[127] = 0;
-  mi_json_buf_print(hbuf, buf);
-}
-
-static void mi_json_buf_print_counter_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_counter_t* stat) {
-  mi_json_buf_print_value(hbuf, name, stat->total);
-}
-
-#define MI_STAT_COUNT(stat)    mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat);
-#define MI_STAT_COUNTER(stat)  mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat);
-
-static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, char* output_buf) mi_attr_noexcept {
-  if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL;
-  mi_json_buf_t hbuf = { NULL, 0, 0, true };
-  if (output_size > 0 && output_buf != NULL) {
-    _mi_memzero(output_buf, output_size);
-    hbuf.buf = output_buf;
-    hbuf.size = output_size;
-    hbuf.can_realloc = false;
-  }
-  else {
-    if (!mi_json_buf_expand(&hbuf)) return NULL;
-  }
-  mi_json_buf_print(&hbuf, "{\n");
-  mi_json_buf_print_value(&hbuf, "stat_version", MI_STAT_VERSION);
-  mi_json_buf_print_value(&hbuf, "mimalloc_version", MI_MALLOC_VERSION);
-
-  // process info
-  mi_json_buf_print(&hbuf, "  \"process\": {\n");
-  size_t elapsed;
-  size_t user_time;
-  size_t sys_time;
-  size_t current_rss;
-  size_t peak_rss;
-  size_t current_commit;
-  size_t peak_commit;
-  size_t page_faults;
-  mi_process_info(&elapsed, &user_time, &sys_time, &current_rss, &peak_rss, &current_commit, &peak_commit, &page_faults);
-  mi_json_buf_print_size(&hbuf, "elapsed_msecs", elapsed, true);
-  mi_json_buf_print_size(&hbuf, "user_msecs", user_time, true);
-  mi_json_buf_print_size(&hbuf, "system_msecs", sys_time, true);
-  mi_json_buf_print_size(&hbuf, "page_faults", page_faults, true);
-  mi_json_buf_print_size(&hbuf, "rss_current", current_rss, true);
-  mi_json_buf_print_size(&hbuf, "rss_peak", peak_rss, true);
-  mi_json_buf_print_size(&hbuf, "commit_current", current_commit, true);
-  mi_json_buf_print_size(&hbuf, "commit_peak", peak_commit, false);
-  mi_json_buf_print(&hbuf, "  },\n");
-
-  // statistics
-  MI_STAT_FIELDS()
-
-  // size bins
-  mi_json_buf_print(&hbuf, "  \"malloc_bins\": [\n");
-  for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
-    mi_json_buf_print_count_bin(&hbuf, "    ", &stats->malloc_bins[i], i, i!=MI_BIN_HUGE);
-  }
-  mi_json_buf_print(&hbuf, "  ],\n");
-  mi_json_buf_print(&hbuf, "  \"page_bins\": [\n");
-  for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
-    mi_json_buf_print_count_bin(&hbuf, "    ", &stats->page_bins[i], i, i!=MI_BIN_HUGE);
-  }
-  mi_json_buf_print(&hbuf, "  ],\n");
-  mi_json_buf_print(&hbuf, "  \"chunk_bins\": [\n");
-  for (size_t i = 0; i < MI_CBIN_COUNT; i++) {
-    mi_json_buf_print_count_cbin(&hbuf, "    ", &stats->chunk_bins[i], (mi_chunkbin_t)i, i!=MI_CBIN_COUNT-1);
-  }
-  mi_json_buf_print(&hbuf, "  ]\n");
-  mi_json_buf_print(&hbuf, "}\n");
-  if (hbuf.used >= hbuf.size) {
-    // failed
-    if (hbuf.can_realloc) { mi_free(hbuf.buf); }
-    return NULL;
-  }
-  else {
-    return hbuf.buf;
-  }
-}
-
-char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept {
-  mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
-  if (subproc==NULL) return NULL;
-  mi_stats_t_decl(stats);
-  if (!mi_subproc_stats_get(subproc_id,&stats)) return NULL;
-  return mi_stats_get_json_from(&stats, buf_size, buf);
-}
-
-char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept {
-  return mi_stats_get_json_from(mi_heap_get_stats(heap), buf_size, buf);
-}
-
-char* mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept {
-  return mi_subproc_stats_get_json(mi_subproc_current(), buf_size, buf);
-}
-
-char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept {
-  return mi_stats_get_json_from(stats, buf_size, buf);
-}
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/theap.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/theap.c
deleted file mode 100644
index ef53c1e0..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/theap.c
+++ /dev/null
@@ -1,715 +0,0 @@
-/*----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"  // _mi_theap_default
-
-#if defined(_MSC_VER) && (_MSC_VER < 1920)
-#pragma warning(disable:4204)  // non-constant aggregate initializer
-#endif
-
-/* -----------------------------------------------------------
-  Helpers
------------------------------------------------------------ */
-
-// return `true` if ok, `false` to break
-typedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2);
-
-// Visit all pages in a theap; returns `false` if break was called.
-static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2)
-{
-  if (theap==NULL || theap->page_count==0) return 0;
-
-  // visit all pages
-  #if MI_DEBUG>1
-  size_t total = theap->page_count;
-  size_t count = 0;
-  #endif
-
-  const size_t max_bin = (include_full ? MI_BIN_FULL : MI_BIN_FULL - 1);
-  for (size_t i = 0; i <= max_bin; i++) {
-    mi_page_queue_t* pq = &theap->pages[i];
-    mi_page_t* page = pq->first;
-    while(page != NULL) {
-      mi_page_t* next = page->next; // save next in case the page gets removed from the queue
-      mi_assert_internal(mi_page_theap(page) == theap);
-      #if MI_DEBUG>1
-      count++;
-      #endif
-      if (!fn(theap, pq, page, arg1, arg2)) return false;
-      page = next; // and continue
-    }
-  }
-  mi_assert_internal(!include_full || count == total);
-  return true;
-}
-
-
-#if MI_DEBUG>=2
-static bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
-  MI_UNUSED(arg1);
-  MI_UNUSED(arg2);
-  MI_UNUSED(pq);
-  mi_assert_internal(mi_page_theap(page) == theap);
-  mi_assert_expensive(_mi_page_is_valid(page));
-  return true;
-}
-#endif
-#if MI_DEBUG>=3
-static bool mi_theap_is_valid(mi_theap_t* theap) {
-  mi_assert_internal(theap!=NULL);
-  mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL);
-  for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
-    mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin]));
-  }
-  return true;
-}
-#endif
-
-
-
-
-/* -----------------------------------------------------------
-  "Collect" pages by migrating `local_free` and `thread_free`
-  lists and freeing empty pages. This is done when a thread
-  stops (and in that case abandons pages if there are still
-  blocks alive)
------------------------------------------------------------ */
-
-typedef enum mi_collect_e {
-  MI_NORMAL,
-  MI_FORCE,
-  MI_ABANDON
-} mi_collect_t;
-
-
-static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {
-  MI_UNUSED(arg2);
-  MI_UNUSED(theap);
-  mi_assert_internal(mi_theap_page_is_valid(theap, pq, page, NULL, NULL));
-  mi_collect_t collect = *((mi_collect_t*)arg_collect);
-  _mi_page_free_collect(page, collect >= MI_FORCE);
-  if (mi_page_all_free(page)) {
-    // no more used blocks, possibly free the page.
-    if (collect >= MI_FORCE || page->retire_expire == 0) {  // either forced/abandon, or not already retired
-      // note: this will potentially free retired pages as well.
-      _mi_page_free(page, pq);
-    }
-  }
-  else if (collect == MI_ABANDON) {
-    // still used blocks but the thread is done; abandon the page
-    _mi_page_abandon(page, pq);
-  }
-  return true; // don't break
-}
-
-static void mi_theap_merge_stats(mi_theap_t* theap) {
-  mi_assert_internal(mi_theap_is_initialized(theap));
-  _mi_stats_merge_into(&_mi_theap_heap(theap)->stats, &theap->stats);
-}
-
-static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect)
-{
-  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
-  mi_assert_expensive(mi_theap_is_valid(theap));
-
-  const bool force = (collect >= MI_FORCE);
-  _mi_deferred_free(theap, force);
-
-  // python/cpython#112532: we may be called from a thread that is not the owner of the theap
-  // const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id());
-
-  // collect retired pages
-  _mi_theap_collect_retired(theap, force);
-
-  // collect all pages owned by this thread
-  mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL);  // dont normally visit full pages, see issue #1220
-
-  // collect arenas (this is program wide so don't force purges on abandonment of threads)
-  //mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1);
-  _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld);
-
-  // merge statistics
-  mi_theap_merge_stats(theap);
-}
-
-void _mi_theap_collect_abandon(mi_theap_t* theap) {
-  mi_theap_collect_ex(theap, MI_ABANDON);
-}
-
-void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept {
-  mi_theap_collect_ex(theap, (force ? MI_FORCE : MI_NORMAL));
-}
-
-void mi_collect(bool force) mi_attr_noexcept {
-  // cannot really collect process wide, just a theap..
-  mi_theap_collect(_mi_theap_default(), force);
-}
-
-void mi_heap_collect(mi_heap_t* heap, bool force) {
-  // cannot really collect a heap, just a theap..
-  mi_theap_collect(mi_heap_theap(heap), force);
-}
-
-/* -----------------------------------------------------------
-  Heap new
------------------------------------------------------------ */
-
-mi_theap_t* mi_theap_get_default(void) {
-  mi_theap_t* theap = _mi_theap_default();
-  if mi_unlikely(!mi_theap_is_initialized(theap)) {
-    mi_thread_init();
-    theap = _mi_theap_default();
-    mi_assert_internal(mi_theap_is_initialized(theap));
-  }
-  return theap;
-}
-
-// todo: make order of parameters consistent (but would that break compat with CPython?)
-void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)
-{
-  mi_assert_internal(theap!=NULL);
-  mi_assert_internal(heap!=NULL);
-  mi_memid_t memid = theap->memid;
-  _mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t));
-  theap->memid = memid;
-  theap->refcount = 1;
-  theap->tld   = tld;  // avoid reading the thread-local tld during initialization
-  mi_atomic_store_ptr_relaxed(mi_heap_t,&theap->heap,heap);
-  mi_assert_internal(theap->stats.size == sizeof(mi_stats_t));
-  
-  _mi_theap_options_init(theap);
-  if (theap->tld->is_in_threadpool) {
-    // if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap.
-    // this is checked in `free.c:mi_free_try_collect_mt`
-    // .. but abandoning is good in this case: halve the full page retain (possibly to 0)
-    // (so blocked threads do not hold on to too much memory)
-    if (theap->page_full_retain > 0) {
-      theap->page_full_retain = theap->page_full_retain / 4;
-    }
-  }
-
-  // push on the thread local theaps list
-  mi_theap_t* head = NULL;
-  mi_lock(&theap->tld->theaps_lock) {
-    head = theap->tld->theaps;
-    theap->tprev = NULL;
-    theap->tnext = head;
-    if (head!=NULL) { head->tprev = theap; }
-    theap->tld->theaps = theap;
-  }
-
-  // initialize random
-  if (head == NULL) {  // first theap in this thread?
-    #if defined(_WIN32) && !defined(MI_SHARED_LIB)
-      _mi_random_init_weak(&theap->random);    // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)
-    #else
-      _mi_random_init(&theap->random);
-    #endif
-  }
-  else {
-    _mi_random_split(&head->random, &theap->random);
-  }
-  theap->cookie  = _mi_theap_random_next(theap) | 1;
-  _mi_theap_guarded_init(theap);
-  mi_subproc_stat_increase(_mi_subproc(),theaps,1);
-
-  // push on the heap's theap list
-  mi_lock(&heap->theaps_lock) {
-    head = heap->theaps;
-    theap->hprev = NULL;
-    theap->hnext = head;
-    if (head!=NULL) { head->hprev = theap; }
-    heap->theaps = theap;
-  }
-}
-
-mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) {
-  mi_assert_internal(tld!=NULL);
-  mi_assert_internal(heap!=NULL);
-  // allocate and initialize a theap
-  mi_memid_t memid;
-  mi_theap_t* theap;
-  //if (!_mi_is_heap_main(heap)) {
-  //  theap = (mi_theap_t*)mi_heap_zalloc(mi_heap_main(),sizeof(mi_theap_t));
-  //  memid = _mi_memid_create(MI_MEM_HEAP_MAIN);
-  //  memid.initially_zero = memid.initially_committed = true;
-  //}
-  //else
-  if (heap->exclusive_arena == NULL) {
-    theap = (mi_theap_t*)_mi_meta_zalloc(sizeof(mi_theap_t), &memid);
-  }
-  else {
-    // theaps associated with a specific arena are allocated in that arena
-    // note: takes up at least one slice which is quite wasteful...
-    const size_t size = _mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE);
-    theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid);
-    mi_assert_internal(memid.mem.os.size >= size);
-  }
-  if (theap==NULL) {
-    _mi_error_message(ENOMEM, "unable to allocate theap meta-data\n");
-    return NULL;
-  }
-  theap->memid = memid;
-  _mi_theap_init(theap, heap, tld);
-  return theap;
-}
-
-uintptr_t _mi_theap_random_next(mi_theap_t* theap) {
-  return _mi_random_next(&theap->random);
-}
-
-static void mi_theap_free_mem(mi_theap_t* theap) {
-  if (theap!=NULL) {
-    mi_subproc_stat_decrease(_mi_subproc(),theaps,1);
-    // free the used memory
-    if (theap->memid.memkind == MI_MEM_HEAP_MAIN) {  // note: for now unused as it would access theap_default stats in mi_free of the current theap
-      mi_assert_internal(_mi_is_heap_main(mi_heap_of(theap)));
-      mi_free(theap);
-    }
-    else if (theap->memid.memkind == MI_MEM_META) {
-      _mi_meta_free(theap, sizeof(*theap), theap->memid);
-    }
-    else {
-      _mi_arenas_free(theap, _mi_align_up(sizeof(*theap),MI_ARENA_MIN_OBJ_SIZE), theap->memid ); // issue #1168, avoid assertion failure
-    }
-  }
-}
-
-void _mi_theap_incref(mi_theap_t* theap) {
-  if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {
-    mi_atomic_increment_acq_rel(&theap->refcount);
-  }
-}
-
-void _mi_theap_decref(mi_theap_t* theap) {
-  if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {
-    if (mi_atomic_decrement_acq_rel(&theap->refcount) == 1) {
-      mi_theap_free_mem(theap);
-    }
-  }
-}
-
-
-// called from `mi_theap_delete` to free the internal theap resources.
-bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock) {
-  mi_assert(theap != NULL);
-  if (theap==NULL) return true;
-
-  mi_heap_t* const heap = mi_atomic_exchange_ptr_acq_rel(mi_heap_t, &theap->heap, NULL);
-  if (heap==NULL) {
-    // concurrent interaction, retry in an outer loop (as the other thread may be blocked on our lock)
-    return false;
-  }
-  else {
-    // merge stats to the owning heap
-    _mi_stats_merge_into(&heap->stats, &theap->stats);
-
-    // remove ourselves from the heap theaps list
-    mi_lock_maybe(&heap->theaps_lock, acquire_heap_theaps_lock) {
-      if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; }
-      if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; }
-                          else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; }
-      theap->hnext = theap->hprev = NULL;
-    }
-
-    // remove ourselves from the thread local theaps list
-    mi_lock_maybe(&theap->tld->theaps_lock, acquire_tld_theaps_lock) {
-      if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev;  }
-      if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext;  }
-                          else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; }
-      theap->tnext = theap->tprev = NULL;                        
-    }
-    theap->tld = NULL;
-    _mi_theap_decref(theap);
-    return true;
-  }
-}
-
-
-/* -----------------------------------------------------------
-  Heap destroy
------------------------------------------------------------ */
-/*
-
-// zero out the page queues
-static void mi_theap_reset_pages(mi_theap_t* theap) {
-  mi_assert_internal(theap != NULL);
-  mi_assert_internal(mi_theap_is_initialized(theap));
-  // TODO: copy full empty theap instead?
-  _mi_memset(&theap->pages_free_direct, 0, sizeof(theap->pages_free_direct));
-  _mi_memcpy_aligned(&theap->pages, &_mi_theap_empty.pages, sizeof(theap->pages));
-  // theap->thread_delayed_free = NULL;
-  theap->page_count = 0;
-}
-
-static bool _mi_theap_page_destroy(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
-  MI_UNUSED(arg1);
-  MI_UNUSED(arg2);
-  MI_UNUSED(pq);
-
-  // ensure no more thread_delayed_free will be added
-  //_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);
-
-  // stats
-  const size_t bsize = mi_page_block_size(page);
-  if (bsize > MI_LARGE_MAX_OBJ_SIZE) {
-    mi_theap_stat_decrease(theap, malloc_huge, bsize);
-  }
-  #if (MI_STAT>0)
-  _mi_page_free_collect(page, false);  // update used count
-  const size_t inuse = page->used;
-  if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
-    mi_theap_stat_decrease(theap, malloc_normal, bsize * inuse);
-    #if (MI_STAT>1)
-    mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], inuse);
-    #endif
-  }
-  // mi_theap_stat_decrease(theap, malloc_requested, bsize * inuse);  // todo: off for aligned blocks...
-  #endif
-
-  /// pretend it is all free now
-  mi_assert_internal(mi_page_thread_free(page) == NULL);
-  page->used = 0;
-
-  // and free the page
-  // mi_page_free(page,false);
-  page->next = NULL;
-  page->prev = NULL;
-  mi_page_set_theap(page, NULL);
-  _mi_arenas_page_free(page, theap);
-
-  return true; // keep going
-}
-
-void _mi_theap_destroy_pages(mi_theap_t* theap) {
-  mi_theap_visit_pages(theap, &_mi_theap_page_destroy, NULL, NULL);
-  mi_theap_reset_pages(theap);
-}
-
-#if MI_TRACK_HEAP_DESTROY
-static bool mi_cdecl mi_theap_track_block_free(const mi_theap_t* theap, const mi_theap_area_t* area, void* block, size_t block_size, void* arg) {
-  MI_UNUSED(theap); MI_UNUSED(area);  MI_UNUSED(arg); MI_UNUSED(block_size);
-  mi_track_free_size(block,mi_usable_size(block));
-  return true;
-}
-#endif
-
-void mi_theap_destroy(mi_theap_t* theap) {
-  mi_assert(theap != NULL);
-  mi_assert(mi_theap_is_initialized(theap));
-  mi_assert(!theap->allow_page_reclaim);
-  mi_assert(!theap->allow_page_abandon);
-  mi_assert_expensive(mi_theap_is_valid(theap));
-  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
-  #if MI_GUARDED
-  // _mi_warning_message("'mi_theap_destroy' called but MI_GUARDED is enabled -- using `mi_theap_delete` instead (theap at %p)\n", theap);
-  mi_theap_delete(theap);
-  return;
-  #else
-  if (theap->allow_page_reclaim) {
-    _mi_warning_message("'mi_theap_destroy' called but ignored as the theap was not created with 'allow_destroy' (theap at %p)\n", theap);
-    // don't free in case it may contain reclaimed pages,
-    mi_theap_delete(theap);
-  }
-  else {
-    // track all blocks as freed
-    #if MI_TRACK_HEAP_DESTROY
-    mi_theap_visit_blocks(theap, true, mi_theap_track_block_free, NULL);
-    #endif
-    // free all pages
-    _mi_theap_destroy_pages(theap);
-    mi_theap_free(theap,true);
-  }
-  #endif
-}
-
-// forcefully destroy all theaps in the current thread
-void _mi_theap_unsafe_destroy_all(mi_theap_t* theap) {
-  mi_assert_internal(theap != NULL);
-  if (theap == NULL) return;
-  mi_theap_t* curr = theap->tld->theaps;
-  while (curr != NULL) {
-    mi_theap_t* next = curr->next;
-    if (!curr->allow_page_reclaim) {
-      mi_theap_destroy(curr);
-    }
-    else {
-      _mi_theap_destroy_pages(curr);
-    }
-    curr = next;
-  }
-}
-*/
-
-/* -----------------------------------------------------------
-  Safe Heap delete
------------------------------------------------------------ */
-
-// Safe delete a theap without freeing any still allocated blocks in that theap.
-void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock)
-{
-  mi_assert(theap != NULL);
-  mi_assert(mi_theap_is_initialized(theap));
-  mi_assert_expensive(mi_theap_is_valid(theap));
-  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
-
-  // abandon all pages
-  _mi_theap_collect_abandon(theap);
-
-  mi_assert_internal(theap->page_count==0);
-  _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock);
-}
-
-
-
-/* -----------------------------------------------------------
-  Load/unload theaps
------------------------------------------------------------ */
-/*
-void mi_theap_unload(mi_theap_t* theap) {
-  mi_assert(mi_theap_is_initialized(theap));
-  mi_assert_expensive(mi_theap_is_valid(theap));
-  if (theap==NULL || !mi_theap_is_initialized(theap)) return;
-  if (_mi_theap_heap(theap)->exclusive_arena == NULL) {
-    _mi_warning_message("cannot unload theaps that are not associated with an exclusive arena\n");
-    return;
-  }
-
-  // abandon all pages so all thread'id in the pages are cleared
-  _mi_theap_collect_abandon(theap);
-  mi_assert_internal(theap->page_count==0);
-
-  // remove from theap list
-  mi_theap_free(theap, false); // but don't actually free the memory
-
-  // disassociate from the current thread-local and static state
-  theap->tld = NULL;
-  return;
-}
-
-bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) {
-  mi_assert(mi_theap_is_initialized(theap));
-  if (theap==NULL || !mi_theap_is_initialized(theap)) return false;
-  if (_mi_theap_heap(theap)->exclusive_arena == NULL) {
-    _mi_warning_message("cannot reload theaps that were not associated with an exclusive arena\n");
-    return false;
-  }
-  if (theap->tld != NULL) {
-    _mi_warning_message("cannot reload theaps that were not unloaded first\n");
-    return false;
-  }
-  mi_arena_t* arena = _mi_arena_from_id(arena_id);
-  if (_mi_theap_heap(theap)->exclusive_arena != arena) {
-    _mi_warning_message("trying to reload a theap at a different arena address: %p vs %p\n", _mi_theap_heap(theap)->exclusive_arena, arena);
-    return false;
-  }
-
-  mi_assert_internal(theap->page_count==0);
-
-  // re-associate with the current thread-local and static state
-  theap->tld = mi_theap_get_default()->tld;
-
-  // reinit direct pages (as we may be in a different process)
-  mi_assert_internal(theap->page_count == 0);
-  for (size_t i = 0; i < MI_PAGES_DIRECT; i++) {
-    theap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty;
-  }
-
-  // push on the thread local theaps list
-  theap->tnext = theap->tld->theaps;
-  theap->tld->theaps = theap;
-  return true;
-}
-*/
-
-
-/* -----------------------------------------------------------
-  Visit all theap blocks and areas
-  Todo: enable visiting abandoned pages, and
-        enable visiting all blocks of all theaps across threads
------------------------------------------------------------ */
-
-void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {
-  const size_t bsize = mi_page_block_size(page);
-  const size_t ubsize = mi_page_usable_block_size(page);
-  area->reserved = page->reserved * bsize;
-  area->committed = page->capacity * bsize;
-  area->blocks = mi_page_start(page);
-  area->used = page->used;   // number of blocks in use (#553)
-  area->block_size = ubsize;
-  area->full_block_size = bsize;
-  area->reserved1 = page;
-}
-
-static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) {
-  mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX);
-  *shift = MI_SIZE_BITS - mi_clz(divisor - 1);
-  *magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);
-}
-
-static size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) {
-  mi_assert_internal(n <= UINT32_MAX);
-  const uint64_t hi = ((uint64_t)n * magic) >> 32;
-  return (size_t)((hi + n) >> shift);
-}
-
-bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) {
-  mi_assert(area != NULL);
-  if (area==NULL) return true;
-  mi_assert(page != NULL);
-  if (page == NULL) return true;
-
-  _mi_page_free_collect(page,true);              // collect both thread_delayed and local_free
-  mi_assert_internal(page->local_free == NULL);
-  if (page->used == 0) return true;
-
-  size_t psize;
-  uint8_t* const pstart = mi_page_area(page, &psize);
-  mi_heap_t* const heap = mi_page_heap(page);
-  const size_t bsize    = mi_page_block_size(page);
-  const size_t ubsize   = mi_page_usable_block_size(page); // without padding
-
-  // optimize page with one block
-  if (page->capacity == 1) {
-    mi_assert_internal(page->used == 1 && page->free == NULL);
-    return visitor(heap, area, pstart, ubsize, arg);
-  }
-  mi_assert(bsize <= UINT32_MAX);
-
-  // optimize full pages
-  if (page->used == page->capacity) {
-    uint8_t* block = pstart;
-    for (size_t i = 0; i < page->capacity; i++) {
-      if (!visitor(heap, area, block, ubsize, arg)) return false;
-      block += bsize;
-    }
-    return true;
-  }
-
-  // create a bitmap of free blocks.
-  #define MI_MAX_BLOCKS   (MI_SMALL_PAGE_SIZE / sizeof(void*))
-  uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS];
-  const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS);
-  memset(free_map, 0, bmapsize * sizeof(intptr_t));
-  if (page->capacity % MI_INTPTR_BITS != 0) {
-    // mark left-over bits at the end as free
-    size_t shift   = (page->capacity % MI_INTPTR_BITS);
-    uintptr_t mask = (UINTPTR_MAX << shift);
-    free_map[bmapsize - 1] = mask;
-  }
-
-  // fast repeated division by the block size
-  uint64_t magic;
-  size_t   shift;
-  mi_get_fast_divisor(bsize, &magic, &shift);
-
-  #if MI_DEBUG>1
-  size_t free_count = 0;
-  #endif
-  for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {
-    #if MI_DEBUG>1
-    free_count++;
-    #endif
-    mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize));
-    size_t offset = (uint8_t*)block - pstart;
-    mi_assert_internal(offset % bsize == 0);
-    mi_assert_internal(offset <= UINT32_MAX);
-    size_t blockidx = mi_fast_divide(offset, magic, shift);
-    mi_assert_internal(blockidx == offset / bsize);
-    mi_assert_internal(blockidx < MI_MAX_BLOCKS);
-    size_t bitidx = (blockidx / MI_INTPTR_BITS);
-    size_t bit = blockidx - (bitidx * MI_INTPTR_BITS);
-    free_map[bitidx] |= ((uintptr_t)1 << bit);
-  }
-  mi_assert_internal(page->capacity == (free_count + page->used));
-
-  // walk through all blocks skipping the free ones
-  #if MI_DEBUG>1
-  size_t used_count = 0;
-  #endif
-  uint8_t* block = pstart;
-  for (size_t i = 0; i < bmapsize; i++) {
-    if (free_map[i] == 0) {
-      // every block is in use
-      for (size_t j = 0; j < MI_INTPTR_BITS; j++) {
-        #if MI_DEBUG>1
-        used_count++;
-        #endif
-        if (!visitor(heap, area, block, ubsize, arg)) return false;
-        block += bsize;
-      }
-    }
-    else {
-      // visit the used blocks in the mask
-      uintptr_t m = ~free_map[i];
-      while (m != 0) {
-        #if MI_DEBUG>1
-        used_count++;
-        #endif
-        size_t bitidx = mi_ctz(m);
-        if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false;
-        m &= m - 1;  // clear least significant bit
-      }
-      block += bsize * MI_INTPTR_BITS;
-    }
-  }
-  mi_assert_internal(page->used == used_count);
-  return true;
-}
-
-
-
-// Separate struct to keep `mi_page_t` out of the public interface
-typedef struct mi_theap_area_ex_s {
-  mi_heap_area_t area;
-  mi_page_t* page;
-} mi_theap_area_ex_t;
-
-typedef bool (mi_theap_area_visit_fun)(const mi_theap_t* theap, const mi_theap_area_ex_t* area, void* arg);
-
-static bool mi_theap_visit_areas_page(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) {
-  MI_UNUSED(theap);
-  MI_UNUSED(pq);
-  mi_theap_area_visit_fun* fun = (mi_theap_area_visit_fun*)vfun;
-  mi_theap_area_ex_t xarea;
-  xarea.page = page;
-  _mi_heap_area_init(&xarea.area, page);
-  return fun(theap, &xarea, arg);
-}
-
-// Visit all theap pages as areas
-static bool mi_theap_visit_areas(const mi_theap_t* theap, mi_theap_area_visit_fun* visitor, void* arg) {
-  if (visitor == NULL) return false;
-  return mi_theap_visit_pages((mi_theap_t*)theap, &mi_theap_visit_areas_page, true, (void*)(visitor), arg); // note: function pointer to void* :-{
-}
-
-// Just to pass arguments
-typedef struct mi_visit_blocks_args_s {
-  bool  visit_blocks;
-  mi_block_visit_fun* visitor;
-  void* arg;
-} mi_visit_blocks_args_t;
-
-static bool mi_theap_area_visitor(const mi_theap_t* theap, const mi_theap_area_ex_t* xarea, void* arg) {
-  mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg;
-  if (!args->visitor(_mi_theap_heap(theap), &xarea->area, NULL, xarea->area.block_size, args->arg)) return false;
-  if (args->visit_blocks) {
-    return _mi_theap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg);
-  }
-  else {
-    return true;
-  }
-}
-
-// Visit all blocks in a theap
-bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
-  mi_visit_blocks_args_t args = { visit_blocks, visitor, arg };
-  return mi_theap_visit_areas(theap, &mi_theap_area_visitor, &args);
-}
-
diff --git a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/threadlocal.c b/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/threadlocal.c
deleted file mode 100644
index eea3329b..00000000
--- a/vendor/libmimalloc-sys/c_src/mimalloc/v3/src/threadlocal.c
+++ /dev/null
@@ -1,236 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2026, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* ----------------------------------------------------------------------------
-Implement dynamic thread local variables (for heap's).
-Unlike most OS native implementations there is no limit on the number
-that can be allocated.
------------------------------------------------------------------------------*/
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"
-
-/* -----------------------------------------------------------
-  Each thread can have (a dynamically expanding) array of
-  thread-local values. Each slot has a value and a version.
-  The version is used to safely reuse slots.
------------------------------------------------------------ */
-typedef struct mi_tls_slot_s {
-  size_t  version;
-  void*   value;
-} mi_tls_slot_t;
-
-typedef struct mi_thread_locals_s {
-  size_t        count;
-  mi_tls_slot_t slots[1];
-} mi_thread_locals_t;
-
-static mi_thread_locals_t mi_thread_locals_empty = { 0, {{0,NULL}} };
-
-mi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty;  // always point to a valid `mi_thread_locals_t`
-
-
-/* -----------------------------------------------------------
-  Each key consists of the slot index in the lower bits,
-  and its version it the top bits. When we get a value
-  the version must match or we return NULL. When we set
-  a value, we also set the version of the key.
------------------------------------------------------------ */
-
-#if MI_SIZE_BITS < 64
-#define MI_TLS_IDX_BITS     (MI_SIZE_BITS/2)      // half for the index, half for the version
-#else
-#define MI_TLS_IDX_BITS     (MI_SIZE_BITS/4)      // 16 bits for the index, 48 bits for the version
-#endif
-#define MI_TLS_IDX_MASK     ((MI_ZU(1)<<MI_TLS_IDX_BITS)-1)
-#define MI_TLS_IDX_MAX      MI_TLS_IDX_MASK
-#define MI_TLS_VERSION_MAX  ((MI_ZU(1)<<(MI_SIZE_BITS - MI_TLS_IDX_BITS))-1)
-
-static size_t mi_key_index( size_t key ) {
-  return (key & MI_TLS_IDX_MASK);
-}
-
-static size_t mi_key_version( size_t key ) {
-  return (key >> MI_TLS_IDX_BITS);
-}
-
-static mi_thread_local_t mi_key_create( size_t index, size_t version ) {
-  mi_assert_internal(version != 0 && version <= MI_TLS_VERSION_MAX);
-  mi_assert_internal(index <= MI_TLS_IDX_MAX);
-  const mi_thread_local_t key = ((version << MI_TLS_IDX_BITS) | index);
-  mi_assert_internal(key != 0);
-  return key;
-}
-
-
-// dynamically reallocate the thread local slots when needed
-static mi_thread_locals_t* mi_thread_locals_expand(size_t least_idx) {
-  mi_thread_locals_t* tls_old = mi_thread_locals;
-  const size_t count_old = tls_old->count;
-  size_t count;
-  if (count_old==0) {
-    tls_old = NULL; // so we allocate fresh from mi_thread_locals_empty
-    count = 16;     // start with 16 slots
-  } 
-  else if (count_old >= 1024) {
-    count = count_old + 1024;  // at some point increase linearly
-  }
-  else {
-    count = 2*count_old;       // and double initially
-  }
-  if (count <= least_idx) {   
-    count = least_idx + 1;
-  }
-  if (count > MI_TLS_IDX_MAX) { return NULL; }  // too large
-  mi_thread_locals_t* tls = (mi_thread_locals_t*)mi_rezalloc(tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t));
-  if mi_unlikely(tls==NULL) return NULL;
-  tls->count = count;
-  mi_thread_locals = tls;
-  return tls;
-}
-
-static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key, void* val ) {
-  if (val==NULL) return true;
-  const size_t idx = mi_key_index(key);  
-  mi_thread_locals_t* tls = mi_thread_locals_expand(idx);
-  if (tls==NULL) return false;
-  mi_assert_internal(tls == mi_thread_locals);
-  mi_assert_internal(idx < tls->count);
-  tls->slots[idx].value = val;
-  tls->slots[idx].version = mi_key_version(key);
-  return true;
-}
-
-// set a tls slot; returns `true` if successful.
-// Can return `false` if we could not reallocate the slots array.
-bool _mi_thread_local_set( mi_thread_local_t key, void* val ) {
-  mi_thread_locals_t* tls = mi_thread_locals;
-  mi_assert_internal(tls!=NULL);
-  mi_assert_internal(key!=0);
-  const size_t idx = mi_key_index(key);
-  if mi_likely(idx < tls->count) {
-    tls->slots[idx].value = val;
-    tls->slots[idx].version = mi_key_version(key);
-    return true;
-  }
-  else {
-    return mi_thread_local_set_expand( key, val );  // tailcall
-  }
-}
-
-// get a tls slot value
-void* _mi_thread_local_get( mi_thread_local_t key ) {
-  const mi_thread_locals_t* const tls = mi_thread_locals;
-  mi_assert_internal(tls!=NULL);
-  mi_assert_internal(key!=0);
-  const size_t idx = mi_key_index(key);
-  if mi_likely(idx < tls->count && mi_key_version(key) == tls->slots[idx].version) {
-    return tls->slots[idx].value;
-  }
-  else {
-    return NULL;  
-  }
-}
-
-void _mi_thread_locals_thread_done(void) {
-  mi_thread_locals_t* const tls = mi_thread_locals;
-  if (tls!=NULL && tls->count > 0) {
-    mi_free(tls);
-    mi_thread_locals = &mi_thread_locals_empty;
-  }
-}
-
-/* -----------------------------------------------------------
-Create and free fresh TLS key's
------------------------------------------------------------ */
-#include "bitmap.h"
-
-static mi_lock_t    mi_thread_locals_lock;    // we need a lock in order to re-allocate the slot bits
-static mi_bitmap_t* mi_thread_locals_free;    // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use)
-static size_t       mi_thread_locals_version; // version to be able to reuse slots safely
-
-void _mi_thread_locals_init(void) {
-  mi_lock_init(&mi_thread_locals_lock);
-}
-
-void _mi_thread_locals_done(void) {
-  mi_lock(&mi_thread_locals_lock) {
-    mi_bitmap_t* const slots = mi_thread_locals_free;
-    mi_free(slots);
-  }
-  mi_lock_done(&mi_thread_locals_lock);
-}
-
-// strange signature but allows us to reuse the arena code for claiming free pages
-static bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, bool* keep_set) {
-  MI_UNUSED(_slice_index); MI_UNUSED(_arena);
-  *keep_set = false;
-  return true;
-}
-
-// When we claim a free slot, we increase the global version counter 
-// (so if we reuse a slot it will be returning NULL initially when a thread tries to get it)
-static mi_thread_local_t mi_thread_local_claim(void) {
-  size_t idx = 0;
-  if (mi_thread_locals_free != NULL && mi_bitmap_try_find_and_claim(mi_thread_locals_free,0,&idx,&mi_thread_local_claim_fun,NULL)) {
-    mi_thread_locals_version++;
-    if (mi_thread_locals_version >= MI_TLS_VERSION_MAX) { mi_thread_locals_version = 1; }  /* wrap around the version */
-    return mi_key_create( idx, mi_thread_locals_version);
-  }
-  else {
-    return 0;
-  }
-}
-
-static bool mi_thread_local_create_expand(void) {
-  mi_bitmap_t* const slots = mi_thread_locals_free;
-  // 1024 bits at a time
-  const size_t oldcount = (slots==NULL ? 0 : mi_bitmap_max_bits(slots));
-  const size_t newcount = 1024 + oldcount;
-  if (newcount > MI_TLS_IDX_MAX) { return false; }
-  const size_t newsize = mi_bitmap_size( newcount, NULL );
-  mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE);
-  if (newslots==NULL) { return false; }
-  if (slots!=NULL) {
-    // copy over the previous bitmap
-    _mi_memcpy_aligned(newslots, slots, mi_bitmap_size(oldcount, NULL)); 
-    mi_free(slots);
-  }
-  mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */);
-  mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount);  /* set the new expanded slots as available */
-  mi_thread_locals_free = newslots;
-  return true;
-}
-
-
-// create a fresh key
-mi_thread_local_t _mi_thread_local_create(void) {
-  mi_thread_local_t key = 0;
-  mi_lock(&mi_thread_locals_lock) {
-    key = mi_thread_local_claim();
-    if (key==0) {
-      if (mi_thread_local_create_expand()) {
-        key = mi_thread_local_claim();
-      }
-    }
-  }
-  return key;
-}
-
-// free a key
-void _mi_thread_local_free(mi_thread_local_t key) {
-  if (key==0) return;
-  const size_t idx = mi_key_index(key);
-  mi_lock(&mi_thread_locals_lock) {
-    mi_bitmap_t* const slots = mi_thread_locals_free;
-    if (slots!=NULL && idx < mi_bitmap_max_bits(slots)) {
-      mi_bitmap_set(slots,idx);
-    }
-  }
-}
-
diff --git a/vendor/libmimalloc-sys/src/extended.rs b/vendor/libmimalloc-sys/src/extended.rs
deleted file mode 100644
index 90a767fc..00000000
--- a/vendor/libmimalloc-sys/src/extended.rs
+++ /dev/null
@@ -1,1126 +0,0 @@
-#![allow(nonstandard_style)]
-
-use core::ffi::c_void;
-
-use cty::{c_char, c_int, c_long, c_ulonglong};
-
-/// The maximum number of bytes which may be used as an argument to a function
-/// in the `_small` family ([`mi_malloc_small`], [`mi_zalloc_small`], etc).
-pub const MI_SMALL_SIZE_MAX: usize = 128 * core::mem::size_of::<*mut c_void>();
-
-extern "C" {
-    /// Allocate `count` items of `size` length each.
-    ///
-    /// Returns `null` if `count * size` overflows or on out-of-memory.
-    ///
-    /// All items are initialized to zero.
-    pub fn mi_calloc(count: usize, size: usize) -> *mut c_void;
-
-    /// Allocate `count` items of `size` length each.
-    ///
-    /// Returns `null` if `count * size` overflows or on out-of-memory,
-    /// otherwise returns the same as [`mi_malloc(count *
-    /// size)`](crate::mi_malloc).
-    /// Equivalent to [`mi_calloc`], but returns uninitialized (and not zeroed)
-    /// bytes.
-    pub fn mi_mallocn(count: usize, size: usize) -> *mut c_void;
-
-    /// Re-allocate memory to `count` elements of `size` bytes.
-    ///
-    /// The realloc equivalent of the [`mi_mallocn`] interface. Returns `null`
-    /// if `count * size` overflows or on out-of-memory, otherwise returns the
-    /// same as [`mi_realloc(p, count * size)`](crate::mi_realloc).
-    pub fn mi_reallocn(p: *mut c_void, count: usize, size: usize) -> *mut c_void;
-
-    /// Try to re-allocate memory to `newsize` bytes _in place_.
-    ///
-    /// Returns null on out-of-memory or if the memory could not be expanded in
-    /// place. On success, returns the same pointer as `p`.
-    ///
-    /// If `newsize` is larger than the original `size` allocated for `p`, the
-    /// bytes after `size` are uninitialized.
-    ///
-    /// If null is returned, the original pointer is not freed.
-    ///
-    /// Note: Conceptually, this is a realloc-like which returns null if it
-    /// would be forced to reallocate memory and copy. In practice it's
-    /// equivalent testing against [`mi_usable_size`](crate::mi_usable_size).
-    pub fn mi_expand(p: *mut c_void, newsize: usize) -> *mut c_void;
-
-    /// Re-allocate memory to `newsize` bytes.
-    ///
-    /// This differs from [`mi_realloc`](crate::mi_realloc) in that on failure,
-    /// `p` is freed.
-    pub fn mi_reallocf(p: *mut c_void, newsize: usize) -> *mut c_void;
-
-    /// Allocate and duplicate a nul-terminated C string.
-    ///
-    /// This can be useful for Rust code when interacting with the FFI.
-    pub fn mi_strdup(s: *const c_char) -> *mut c_char;
-
-    /// Allocate and duplicate a nul-terminated C string, up to `n` bytes.
-    ///
-    /// This can be useful for Rust code when interacting with the FFI.
-    pub fn mi_strndup(s: *const c_char, n: usize) -> *mut c_char;
-
-    /// Resolve a file path name, producing a `C` string which can be passed to
-    /// [`mi_free`](crate::mi_free).
-    ///
-    /// `resolved_name` should be null, but can also point to a buffer of at
-    /// least `PATH_MAX` bytes.
-    ///
-    /// If successful, returns a pointer to the resolved absolute file name, or
-    /// `null` on failure (with `errno` set to the error code).
-    ///
-    /// If `resolved_name` was `null`, the returned result should be freed with
-    /// [`mi_free`](crate::mi_free).
-    ///
-    /// This can rarely be useful in FFI code, but is mostly included for
-    /// completeness.
-    pub fn mi_realpath(fname: *const c_char, resolved_name: *mut c_char) -> *mut c_char;
-
-    /// Allocate `size * count` bytes aligned by `alignment`.
-    ///
-    /// Return pointer to the allocated memory or null if out of memory or if
-    /// `size * count` overflows.
-    ///
-    /// Returns a unique pointer if called with `size * count` 0.
-    pub fn mi_calloc_aligned(count: usize, size: usize, alignment: usize) -> *mut c_void;
-
-    /// Allocate `size` bytes aligned by `alignment` at a specified `offset`.
-    ///
-    /// Note that the resulting pointer itself is not aligned by the alignment,
-    /// but after `offset` bytes it will be. This can be useful for allocating
-    /// data with an inline header, where the data has a specific alignment
-    /// requirement.
-    ///
-    /// Specifically, if `p` is the returned pointer `p.add(offset)` is aligned
-    /// to `alignment`.
-    pub fn mi_malloc_aligned_at(size: usize, alignment: usize, offset: usize) -> *mut c_void;
-
-    /// Allocate `size` bytes aligned by `alignment` at a specified `offset`,
-    /// zero-initialized.
-    ///
-    /// This is a [`mi_zalloc`](crate::mi_zalloc) equivalent of [`mi_malloc_aligned_at`].
-    pub fn mi_zalloc_aligned_at(size: usize, alignment: usize, offset: usize) -> *mut c_void;
-
-    /// Allocate `size` of bytes aligned by `alignment` and place the address of the
-    /// allocated memory to `ptr`.
-    ///
-    /// Returns zero on success, invalid argument for invalid alignment, or out-of-memory.
-    pub fn mi_posix_memalign(ptr: *mut *mut c_void, alignment: usize, size: usize) -> c_int;
-
-    /// Allocate `size` bytes aligned by `alignment` with alignment as the first
-    /// parameter.
-    ///
-    /// Return pointer to the allocated memory or null if out of memory.
-    pub fn mi_aligned_alloc(alignment: usize, size: usize) -> *mut c_void;
-
-    /// Allocate `size * count` bytes aligned by `alignment` at a specified
-    /// `offset`, zero-initialized.
-    ///
-    /// This is a [`calloc`](crate::mi_calloc) equivalent of [`mi_malloc_aligned_at`].
-    pub fn mi_calloc_aligned_at(
-        count: usize,
-        size: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Re-allocate memory to `newsize` bytes aligned by `alignment` at a
-    /// specified `offset`.
-    ///
-    /// This is a [`realloc`](crate::mi_realloc) equivalent of [`mi_malloc_aligned_at`].
-    pub fn mi_realloc_aligned_at(
-        p: *mut c_void,
-        newsize: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Zero initialized [re-allocation](crate::mi_realloc).
-    ///
-    /// In general, only valid on memory originally allocated by zero
-    /// initialization: [`mi_calloc`](crate::mi_calloc),
-    /// [`mi_zalloc`](crate::mi_zalloc),
-    /// [`mi_zalloc_aligned`](crate::mi_zalloc_aligned), ...
-    pub fn mi_rezalloc(p: *mut c_void, newsize: usize) -> *mut c_void;
-
-    /// Zero initialized [re-allocation](crate::mi_realloc), following `calloc`
-    /// paramater conventions.
-    ///
-    /// In general, only valid on memory originally allocated by zero
-    /// initialization: [`mi_calloc`](crate::mi_calloc),
-    /// [`mi_zalloc`](crate::mi_zalloc),
-    /// [`mi_zalloc_aligned`](crate::mi_zalloc_aligned), ...
-    pub fn mi_recalloc(p: *mut c_void, newcount: usize, size: usize) -> *mut c_void;
-
-    /// Aligned version of [`mi_rezalloc`].
-    pub fn mi_rezalloc_aligned(p: *mut c_void, newsize: usize, alignment: usize) -> *mut c_void;
-
-    /// Offset-aligned version of [`mi_rezalloc`].
-    pub fn mi_rezalloc_aligned_at(
-        p: *mut c_void,
-        newsize: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Aligned version of [`mi_recalloc`].
-    pub fn mi_recalloc_aligned(
-        p: *mut c_void,
-        newcount: usize,
-        size: usize,
-        alignment: usize,
-    ) -> *mut c_void;
-
-    /// Offset-aligned version of [`mi_recalloc`].
-    pub fn mi_recalloc_aligned_at(
-        p: *mut c_void,
-        newcount: usize,
-        size: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Allocate an object of no more than [`MI_SMALL_SIZE_MAX`] bytes.
-    ///
-    /// Does not check that `size` is indeed small.
-    ///
-    /// Note: Currently [`mi_malloc`](crate::mi_malloc) checks if `size` is
-    /// small and calls this if
-    /// so at runtime, so its' only worth using if you know for certain.
-    pub fn mi_malloc_small(size: usize) -> *mut c_void;
-
-    /// Allocate an zero-initialized object of no more than
-    /// [`MI_SMALL_SIZE_MAX`] bytes.
-    ///
-    /// Does not check that `size` is indeed small.
-    ///
-    /// Note: Currently [`mi_zalloc`](crate::mi_zalloc) checks if `size` is
-    /// small and calls this if so at runtime, so its' only worth using if you
-    /// know for certain.
-    pub fn mi_zalloc_small(size: usize) -> *mut c_void;
-
-    /// Free a small object. Only use to free objects from [`mi_malloc_small`]
-    /// or [`mi_zalloc_small`]. Potentially a tiny bit faster than [`mi_free`](crate::mi_free).
-    pub fn mi_free_small(p: *mut c_void);
-
-    /// Return the available bytes in a memory block.
-    ///
-    /// The returned size can be used to call `mi_expand` successfully.
-    pub fn mi_usable_size(p: *const c_void) -> usize;
-
-    /// Return the used allocation size.
-    ///
-    /// Returns the size `n` that will be allocated, where `n >= size`.
-    ///
-    /// Generally, `mi_usable_size(mi_malloc(size)) == mi_good_size(size)`. This
-    /// can be used to reduce internal wasted space when allocating buffers for
-    /// example.
-    ///
-    /// See [`mi_usable_size`](crate::mi_usable_size).
-    pub fn mi_good_size(size: usize) -> usize;
-
-    /// Eagerly free memory.
-    ///
-    /// If `force` is true, aggressively return memory to the OS (can be
-    /// expensive!)
-    ///
-    /// Regular code should not have to call this function. It can be beneficial
-    /// in very narrow circumstances; in particular, when a long running thread
-    /// allocates a lot of blocks that are freed by other threads it may improve
-    /// resource usage by calling this every once in a while.
-    pub fn mi_collect(force: bool);
-
-    /// Checked free: If `p` came from mimalloc's heap (as decided by
-    /// [`mi_is_in_heap_region`]), this is [`mi_free(p)`](crate::mi_free), but
-    /// otherwise it is a no-op.
-    pub fn mi_cfree(p: *mut c_void);
-
-    /// Returns true if this is a pointer into a memory region that has been
-    /// reserved by the mimalloc heap.
-    ///
-    /// This function is described by the mimalloc documentation as "relatively
-    /// fast".
-    ///
-    /// See also [`mi_heap_check_owned`], which is (much) slower and slightly
-    /// more precise, but only concerns a single `mi_heap`.
-    pub fn mi_is_in_heap_region(p: *const c_void) -> bool;
-
-    /// Layout-aware deallocation: Like [`mi_free`](crate::mi_free), but accepts
-    /// the size and alignment as well.
-    ///
-    /// Note: unlike some allocators that require this information for
-    /// performance, mimalloc doesn't need it (as of the current version,
-    /// v2.0.0), and so it currently implements this as a (debug) assertion that
-    /// verifies that `p` is actually aligned to `alignment` and is usable for
-    /// at least `size` bytes, before delegating to `mi_free`.
-    ///
-    /// However, currently there's no way to have this crate enable mimalloc's
-    /// debug assertions, so these checks aren't particularly useful.
-    ///
-    /// Note: It's legal to pass null to this function, and you are not required
-    /// to use this to deallocate memory from an aligned allocation function.
-    pub fn mi_free_size_aligned(p: *mut c_void, size: usize, alignment: usize);
-
-    /// Size-aware deallocation: Like [`mi_free`](crate::mi_free), but accepts
-    /// the size and alignment as well.
-    ///
-    /// Note: unlike some allocators that require this information for
-    /// performance, mimalloc doesn't need it (as of the current version,
-    /// v2.0.0), and so it currently implements this as a (debug) assertion that
-    /// verifies that `p` is actually aligned to `alignment` and is usable for
-    /// at least `size` bytes, before delegating to `mi_free`.
-    ///
-    /// However, currently there's no way to have this crate enable mimalloc's
-    /// debug assertions, so these checks aren't particularly useful.
-    ///
-    /// Note: It's legal to pass null to this function.
-    pub fn mi_free_size(p: *mut c_void, size: usize);
-
-    /// Alignment-aware deallocation: Like [`mi_free`](crate::mi_free), but
-    /// accepts the size and alignment as well.
-    ///
-    /// Note: unlike some allocators that require this information for
-    /// performance, mimalloc doesn't need it (as of the current version,
-    /// v2.0.0), and so it currently implements this as a (debug) assertion that
-    /// verifies that `p` is actually aligned to `alignment` and is usable for
-    /// at least `size` bytes, before delegating to `mi_free`.
-    ///
-    /// However, currently there's no way to have this crate enable mimalloc's
-    /// debug assertions, so these checks aren't particularly useful.
-    ///
-    /// Note: It's legal to pass null to this function.
-    pub fn mi_free_aligned(p: *mut c_void, alignment: usize);
-
-    /// Print the main statistics.
-    ///
-    /// Ignores the passed in argument, and outputs to the registered output
-    /// function or stderr by default.
-    ///
-    /// Most detailed when using a debug build.
-    pub fn mi_stats_print(_: *mut c_void);
-
-    /// Print the main statistics.
-    ///
-    /// Pass `None` for `out` to use the default. If `out` is provided, `arc` is
-    /// passed as it's second parameter.
-    ///
-    /// Most detailed when using a debug build.
-    pub fn mi_stats_print_out(out: mi_output_fun, arg: *mut c_void);
-
-    /// Reset statistics.
-    ///
-    /// Note: This function is thread safe.
-    pub fn mi_stats_reset();
-
-    /// Merge thread local statistics with the main statistics and reset.
-    ///
-    /// Note: This function is thread safe.
-    #[cfg(feature = "v2")]
-    pub fn mi_stats_merge();
-
-    /// Return the mimalloc version number.
-    ///
-    /// For example version 1.6.3 would return the number `163`.
-    pub fn mi_version() -> c_int;
-
-    /// Initialize mimalloc on a thread.
-    ///
-    /// Should not be used as on most systems (pthreads, windows) this is done
-    /// automatically.
-    pub fn mi_thread_init();
-
-    /// Initialize the process.
-    ///
-    /// Should not be used on most systems, as it's called by thread_init or the
-    /// process loader.
-    pub fn mi_process_init();
-
-    /// Return process information (time and memory usage). All parameters are
-    /// optional (nullable) out-params:
-    ///
-    /// | Parameter        | Description |
-    /// | :-               | :- |
-    /// | `elapsed_msecs`  | Elapsed wall-clock time of the process in milli-seconds. |
-    /// | `user_msecs`     | User time in milli-seconds (as the sum over all threads). |
-    /// | `system_msecs`   | System time in milli-seconds. |
-    /// | `current_rss`    | Current working set size (touched pages). |
-    /// | `peak_rss`       | Peak working set size (touched pages). |
-    /// | `current_commit` | Current committed memory (backed by the page file). |
-    /// | `peak_commit`    | Peak committed memory (backed by the page file). |
-    /// | `page_faults`    | Count of hard page faults. |
-    ///
-    /// The `current_rss` is precise on Windows and MacOSX; other systems
-    /// estimate this using `current_commit`. The `commit` is precise on Windows
-    /// but estimated on other systems as the amount of read/write accessible
-    /// memory reserved by mimalloc.
-    pub fn mi_process_info(
-        elapsed_msecs: *mut usize,
-        user_msecs: *mut usize,
-        system_msecs: *mut usize,
-        current_rss: *mut usize,
-        peak_rss: *mut usize,
-        current_commit: *mut usize,
-        peak_commit: *mut usize,
-        page_faults: *mut usize,
-    );
-
-    /// Uninitialize mimalloc on a thread.
-    ///
-    /// Should not be used as on most systems (pthreads, windows) this is done
-    /// automatically. Ensures that any memory that is not freed yet (but will
-    /// be freed by other threads in the future) is properly handled.
-    ///
-    /// Note: This function is thread safe.
-    pub fn mi_thread_done();
-
-    /// Print out heap statistics for this thread.
-    ///
-    /// Pass `None` for `out` to use the default. If `out` is provided, `arc` is
-    /// passed as it's second parameter
-    ///
-    /// Most detailed when using a debug build.
-    ///
-    /// Note: This function is thread safe.
-    pub fn mi_thread_stats_print_out(out: mi_output_fun, arg: *mut c_void);
-
-    /// Register an output function.
-    ///
-    /// - `out` The output function, use `None` to output to stderr.
-    /// - `arg` Argument that will be passed on to the output function.
-    ///
-    /// The `out` function is called to output any information from mimalloc,
-    /// like verbose or warning messages.
-    ///
-    /// Note: This function is thread safe.
-    pub fn mi_register_output(out: mi_output_fun, arg: *mut c_void);
-
-    /// Register a deferred free function.
-    ///
-    /// - `deferred_free` Address of a deferred free-ing function or `None` to
-    ///   unregister.
-    /// - `arg` Argument that will be passed on to the deferred free function.
-    ///
-    /// Some runtime systems use deferred free-ing, for example when using
-    /// reference counting to limit the worst case free time.
-    ///
-    /// Such systems can register (re-entrant) deferred free function to free
-    /// more memory on demand.
-    ///
-    /// - When the `force` parameter is `true` all possible memory should be
-    ///   freed.
-    ///
-    /// - The per-thread `heartbeat` parameter is monotonically increasing and
-    ///   guaranteed to be deterministic if the program allocates
-    ///   deterministically.
-    ///
-    /// - The `deferred_free` function is guaranteed to be called
-    ///   deterministically after some number of allocations (regardless of
-    ///   freeing or available free memory).
-    ///
-    /// At most one `deferred_free` function can be active.
-    ///
-    /// Note: This function is thread safe.
-    pub fn mi_register_deferred_free(out: mi_deferred_free_fun, arg: *mut c_void);
-
-    /// Register an error callback function.
-    ///
-    /// The `errfun` function is called on an error in mimalloc after emitting
-    /// an error message (through the output function).
-    ///
-    /// It as always legal to just return from the `errfun` function in which
-    /// case allocation functions generally return null or ignore the condition.
-    ///
-    /// The default function only calls abort() when compiled in secure mode
-    /// with an `EFAULT` error. The possible error codes are:
-    ///
-    /// - `EAGAIN` (11): Double free was detected (only in debug and secure
-    ///   mode).
-    /// - `EFAULT` (14): Corrupted free list or meta-data was detected (only in
-    ///   debug and secure mode).
-    /// - `ENOMEM` (12): Not enough memory available to satisfy the request.
-    /// - `EOVERFLOW` (75): Too large a request, for example in `mi_calloc`, the
-    ///   `count` and `size` parameters are too large.
-    /// - `EINVAL` (22): Trying to free or re-allocate an invalid pointer.
-    ///
-    /// Note: This function is thread safe.
-    pub fn mi_register_error(out: mi_error_fun, arg: *mut c_void);
-
-    /// Get the statistics for the current subprocess aggregated over all its heaps as JSON.
-    ///
-    /// Returns pointer to the buffer or NULL on failure. Use mi_free() to free the buffer if the buf parameter was NULL.
-    #[cfg(not(feature = "v2"))]
-    pub fn mi_stats_get_json(buf_size: usize, buf: *mut c_char) -> *mut c_char;
-}
-
-/// An output callback. Must be thread-safe.
-///
-/// See [`mi_stats_print_out`], [`mi_thread_stats_print_out`], [`mi_register_output`]
-pub type mi_output_fun = Option<unsafe extern "C" fn(msg: *const c_char, arg: *mut c_void)>;
-
-/// Type of deferred free functions. Must be thread-safe.
-///
-/// - `force`: If true, all outstanding items should be freed.
-/// - `heartbeat` A monotonically increasing count.
-/// - `arg` Argument that was passed at registration to hold extra state.
-///
-/// See [`mi_register_deferred_free`]
-pub type mi_deferred_free_fun =
-    Option<unsafe extern "C" fn(force: bool, heartbeat: c_ulonglong, arg: *mut c_void)>;
-
-/// Type of error callback functions. Must be thread-safe.
-///
-/// - `err`: Error code (see [`mi_register_error`] for a list).
-/// - `arg`: Argument that was passed at registration to hold extra state.
-///
-/// See [`mi_register_error`]
-pub type mi_error_fun = Option<unsafe extern "C" fn(code: c_int, arg: *mut c_void)>;
-
-/// Runtime options. All options are false by default.
-pub type mi_option_t = c_int;
-
-#[cfg(all(feature = "arena", feature = "v2"))]
-/// Arena Id
-pub type mi_arena_id_t = c_int;
-
-#[cfg(all(feature = "arena", not(feature = "v2")))]
-/// Arena Id
-pub type mi_arena_id_t = *mut c_void;
-
-// Note: mimalloc doc website seems to have the order of show_stats and
-// show_errors reversed as of 1.6.3, however what I have here is correct:
-// https://github.com/microsoft/mimalloc/issues/266#issuecomment-653822341
-
-/// Print error messages to `stderr`.
-pub const mi_option_show_errors: mi_option_t = 0;
-
-/// Print statistics to `stderr` when the program is done.
-pub const mi_option_show_stats: mi_option_t = 1;
-
-/// Print verbose messages to `stderr`.
-pub const mi_option_verbose: mi_option_t = 2;
-
-/// ### The following options are experimental
-///
-/// Option (experimental) Use large OS pages (2MiB in size) if possible.
-///
-/// Use large OS pages (2MiB) when available; for some workloads this can
-/// significantly improve performance. Use mi_option_verbose to check if
-/// the large OS pages are enabled -- usually one needs to explicitly allow
-/// large OS pages (as on Windows and Linux). However, sometimes the OS is
-/// very slow to reserve contiguous physical memory for large OS pages so
-/// use with care on systems that can have fragmented memory (for that
-/// reason, we generally recommend to use mi_option_reserve_huge_os_pages
-/// instead whenever possible).
-pub const mi_option_large_os_pages: mi_option_t = 6;
-
-/// Option (experimental) The number of huge OS pages (1GiB in size) to reserve at the start of the program.
-///
-/// This reserves the huge pages at startup and sometimes this can give a large (latency) performance
-/// improvement on big workloads. Usually it is better to not use MIMALLOC_LARGE_OS_PAGES in
-/// combination with this setting. Just like large OS pages, use with care as reserving contiguous
-/// physical memory can take a long time when memory is fragmented (but reserving the huge pages is
-/// done at startup only once). Note that we usually need to explicitly enable huge OS pages (as on
-/// Windows and Linux)). With huge OS pages, it may be beneficial to set the setting
-/// mi_option_eager_commit_delay=N (N is 1 by default) to delay the initial N segments (of 4MiB) of
-/// a thread to not allocate in the huge OS pages; this prevents threads that are short lived and
-/// allocate just a little to take up space in the huge OS page area (which cannot be reset).
-pub const mi_option_reserve_huge_os_pages: mi_option_t = 7;
-
-/// Option (experimental) Reserve huge OS pages at node N.
-///
-/// The huge pages are usually allocated evenly among NUMA nodes.
-/// You can use mi_option_reserve_huge_os_pages_at=N where `N` is the numa node (starting at 0) to allocate all
-/// the huge pages at a specific numa node instead.
-pub const mi_option_reserve_huge_os_pages_at: mi_option_t = 8;
-
-/// Option (experimental) Reserve specified amount of OS memory at startup, e.g. "1g" or "512m".
-pub const mi_option_reserve_os_memory: mi_option_t = 9;
-
-#[cfg(feature = "v2")]
-/// Option (experimental) the first N segments per thread are not eagerly committed (=1).
-pub const mi_option_eager_commit_delay: mi_option_t = 14;
-
-/// Option (experimental) Pretend there are at most N NUMA nodes; Use 0 to use the actual detected NUMA nodes at runtime.
-pub const mi_option_use_numa_nodes: mi_option_t = 16;
-
-/// Option (experimental) If set to 1, do not use OS memory for allocation (but only pre-reserved arenas)
-pub const mi_option_limit_os_alloc: mi_option_t = 17;
-
-/// Option (experimental) OS tag to assign to mimalloc'd memory
-pub const mi_option_os_tag: mi_option_t = 18;
-
-/// Option (experimental)
-pub const mi_option_max_errors: mi_option_t = 19;
-
-/// Option (experimental)
-pub const mi_option_max_warnings: mi_option_t = 20;
-
-#[cfg(feature = "v2")]
-/// Option (experimental)
-pub const mi_option_max_segment_reclaim: mi_option_t = 21;
-
-/// Last option.
-#[cfg(feature = "v2")]
-pub const _mi_option_last: mi_option_t = 38;
-#[cfg(not(feature = "v2"))]
-pub const _mi_option_last: mi_option_t = 47;
-
-extern "C" {
-    // Note: mi_option_{enable,disable} aren't exposed because they're redundant
-    // and because of https://github.com/microsoft/mimalloc/issues/266.
-
-    /// Returns true if the provided option is enabled.
-    ///
-    /// Note: this function is not thread safe.
-    pub fn mi_option_is_enabled(option: mi_option_t) -> bool;
-
-    /// Enable or disable the given option.
-    ///
-    /// Note: this function is not thread safe.
-    pub fn mi_option_set_enabled(option: mi_option_t, enable: bool);
-
-    /// If the given option has not yet been initialized with [`mi_option_set`]
-    /// or [`mi_option_set_enabled`], enables or disables the option. If it has,
-    /// this function does nothing.
-    ///
-    /// Note: this function is not thread safe.
-    pub fn mi_option_set_enabled_default(option: mi_option_t, enable: bool);
-
-    /// Returns the value of the provided option.
-    ///
-    /// The value of boolean options is 1 or 0, however experimental options
-    /// exist which take a numeric value, which is the intended use of this
-    /// function.
-    ///
-    /// These options are not exposed as constants for stability reasons,
-    /// however you can still use them as arguments to this and other
-    /// `mi_option_` functions if needed, see the mimalloc documentation for
-    /// details: https://microsoft.github.io/mimalloc/group__options.html
-    ///
-    /// Note: this function is not thread safe.
-    pub fn mi_option_get(option: mi_option_t) -> c_long;
-
-    /// Set the option to the given value.
-    ///
-    /// The value of boolean options is 1 or 0, however experimental options
-    /// exist which take a numeric value, which is the intended use of this
-    /// function.
-    ///
-    /// These options are not exposed as constants for stability reasons,
-    /// however you can still use them as arguments to this and other
-    /// `mi_option_` functions if needed,
-    ///
-    /// Note: this function is not thread safe.
-    pub fn mi_option_set(option: mi_option_t, value: c_long);
-
-    /// If the given option has not yet been initialized with [`mi_option_set`]
-    /// or [`mi_option_set_enabled`], sets the option to the given value. If it
-    /// has, this function does nothing.
-    ///
-    /// The value of boolean options is 1 or 0, however experimental options
-    /// exist which take a numeric value, which is the intended use of this
-    /// function.
-    ///
-    /// These options are not exposed as constants for stability reasons,
-    /// however you can still use them as arguments to this and other
-    /// `mi_option_` functions if needed.
-    ///
-    /// Note: this function is not thread safe.
-    pub fn mi_option_set_default(option: mi_option_t, value: c_long);
-}
-
-/// First-class heaps that can be destroyed in one go.
-///
-/// Note: The pointers allocated out of a heap can be be freed using
-/// [`mi_free`](crate::mi_free) -- there is no `mi_heap_free`.
-///
-/// # Example
-///
-/// ```
-/// use libmimalloc_sys as mi;
-/// unsafe {
-///     let h = mi::mi_heap_new();
-///     assert!(!h.is_null());
-///     let p = mi::mi_heap_malloc(h, 50);
-///     assert!(!p.is_null());
-///
-///     // use p...
-///     mi::mi_free(p);
-///
-///     // Clean up the heap. Note that pointers allocated from `h`
-///     // are *not* invalided by `mi_heap_delete`. You would have
-///     // to use (the very dangerous) `mi_heap_destroy` for that
-///     // behavior
-///     mi::mi_heap_delete(h);
-/// }
-/// ```
-pub enum mi_heap_t {}
-
-#[cfg(not(feature = "v2"))]
-pub enum mi_theap_t {}
-
-/// An area of heap space contains blocks of a single size.
-///
-/// The bytes in freed blocks are `committed - used`.
-#[repr(C)]
-#[derive(Debug, Clone, Copy)]
-pub struct mi_heap_area_t {
-    /// Start of the area containing heap blocks.
-    pub blocks: *mut c_void,
-    /// Bytes reserved for this area.
-    pub reserved: usize,
-    /// Current committed bytes of this area.
-    pub committed: usize,
-    /// Bytes in use by allocated blocks.
-    pub used: usize,
-    /// Size in bytes of one block.
-    pub block_size: usize,
-    /// Size in bytes of a full block including padding and metadata.
-    pub full_block_size: usize,
-    /// Heap tag associated with this area (not available in v3)
-    #[cfg(feature = "v2")]
-    pub heap_tag: i32,
-    /// Reserved / internal (replaces `heap_tag` in v3)
-    #[cfg(not(feature = "v2"))]
-    pub reserved1: *mut c_void,
-}
-
-/// Visitor function passed to [`mi_heap_visit_blocks`]
-///
-/// Should return `true` to continue, and `false` to stop visiting (i.e. break)
-///
-/// This function is always first called for every `area` with `block` as a null
-/// pointer. If `visit_all_blocks` was `true`, the function is then called for
-/// every allocated block in that area.
-pub type mi_block_visit_fun = Option<
-    unsafe extern "C" fn(
-        heap: *const mi_heap_t,
-        area: *const mi_heap_area_t,
-        block: *mut c_void,
-        block_size: usize,
-        arg: *mut c_void,
-    ) -> bool,
->;
-
-extern "C" {
-    /// Create a new heap that can be used for allocation.
-    pub fn mi_heap_new() -> *mut mi_heap_t;
-
-    /// Delete a previously allocated heap.
-    ///
-    /// This will release resources and migrate any still allocated blocks in
-    /// this heap (efficienty) to the default heap.
-    ///
-    /// If `heap` is the default heap, the default heap is set to the backing
-    /// heap.
-    pub fn mi_heap_delete(heap: *mut mi_heap_t);
-
-    /// Destroy a heap, freeing all its still allocated blocks.
-    ///
-    /// Use with care as this will free all blocks still allocated in the heap.
-    /// However, this can be a very efficient way to free all heap memory in one
-    /// go.
-    ///
-    /// If `heap` is the default heap, the default heap is set to the backing
-    /// heap.
-    pub fn mi_heap_destroy(heap: *mut mi_heap_t);
-
-    /// Set the default heap to use for [`mi_malloc`](crate::mi_malloc) et al.
-    ///
-    /// Returns the previous default heap.
-    #[cfg(feature = "v2")]
-    pub fn mi_heap_set_default(heap: *mut mi_heap_t) -> *mut mi_heap_t;
-
-    /// Get the default heap that is used for [`mi_malloc`](crate::mi_malloc) et al.
-    #[cfg(feature = "v2")]
-    pub fn mi_heap_get_default() -> *mut mi_heap_t;
-
-    /// Get the backing heap.
-    ///
-    /// The _backing_ heap is the initial default heap for a thread and always
-    /// available for allocations. It cannot be destroyed or deleted except by
-    /// exiting the thread.
-    #[cfg(feature = "v2")]
-    pub fn mi_heap_get_backing() -> *mut mi_heap_t;
-
-    /// Release outstanding resources in a specific heap.
-    ///
-    /// See also [`mi_collect`].
-    pub fn mi_heap_collect(heap: *mut mi_heap_t, force: bool);
-
-    /// Equivalent to [`mi_malloc`](crate::mi_malloc), but allocates out of the
-    /// specific heap instead of the default.
-    pub fn mi_heap_malloc(heap: *mut mi_heap_t, size: usize) -> *mut c_void;
-
-    /// Equivalent to [`mi_zalloc`](crate::mi_zalloc), but allocates out of the
-    /// specific heap instead of the default.
-    pub fn mi_heap_zalloc(heap: *mut mi_heap_t, size: usize) -> *mut c_void;
-
-    /// Equivalent to [`mi_calloc`], but allocates out of the specific heap
-    /// instead of the default.
-    pub fn mi_heap_calloc(heap: *mut mi_heap_t, count: usize, size: usize) -> *mut c_void;
-
-    /// Equivalent to [`mi_mallocn`], but allocates out of the specific heap
-    /// instead of the default.
-    pub fn mi_heap_mallocn(heap: *mut mi_heap_t, count: usize, size: usize) -> *mut c_void;
-
-    /// Equivalent to [`mi_malloc_small`], but allocates out of the specific
-    /// heap instead of the default.
-    ///
-    /// `size` must be smaller or equal to [`MI_SMALL_SIZE_MAX`].
-    pub fn mi_heap_malloc_small(heap: *mut mi_heap_t, size: usize) -> *mut c_void;
-
-    /// Equivalent to [`mi_zalloc_small`], but allocates out of the specific
-    /// heap instead of the default.
-    ///
-    /// `size` must be smaller or equal to [`MI_SMALL_SIZE_MAX`].
-    pub fn mi_heap_zalloc_small(heap: *mut mi_heap_t, size: usize) -> *mut c_void;
-
-    /// Equivalent to [`mi_realloc`](crate::mi_realloc), but allocates out of
-    /// the specific heap instead of the default.
-    pub fn mi_heap_realloc(heap: *mut mi_heap_t, p: *mut c_void, newsize: usize) -> *mut c_void;
-
-    /// Equivalent to [`mi_reallocn`], but allocates out of the specific heap
-    /// instead of the default.
-    pub fn mi_heap_reallocn(
-        heap: *mut mi_heap_t,
-        p: *mut c_void,
-        count: usize,
-        size: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_reallocf`], but allocates out of the specific heap
-    /// instead of the default.
-    pub fn mi_heap_reallocf(heap: *mut mi_heap_t, p: *mut c_void, newsize: usize) -> *mut c_void;
-
-    /// Equivalent to [`mi_strdup`], but allocates out of the specific heap
-    /// instead of the default.
-    pub fn mi_heap_strdup(heap: *mut mi_heap_t, s: *const c_char) -> *mut c_char;
-
-    /// Equivalent to [`mi_strndup`], but allocates out of the specific heap
-    /// instead of the default.
-    pub fn mi_heap_strndup(heap: *mut mi_heap_t, s: *const c_char, n: usize) -> *mut c_char;
-
-    /// Equivalent to [`mi_realpath`], but allocates out of the specific heap
-    /// instead of the default.
-    pub fn mi_heap_realpath(
-        heap: *mut mi_heap_t,
-        fname: *const c_char,
-        resolved_name: *mut c_char,
-    ) -> *mut c_char;
-
-    /// Equivalent to [`mi_malloc_aligned`](crate::mi_malloc_aligned), but
-    /// allocates out of the specific heap instead of the default.
-    pub fn mi_heap_malloc_aligned(
-        heap: *mut mi_heap_t,
-        size: usize,
-        alignment: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_malloc_aligned_at`], but allocates out of the
-    /// specific heap instead of the default.
-    pub fn mi_heap_malloc_aligned_at(
-        heap: *mut mi_heap_t,
-        size: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_zalloc_aligned`](crate::mi_zalloc_aligned), but
-    /// allocates out of the specific heap instead of the default.
-    pub fn mi_heap_zalloc_aligned(
-        heap: *mut mi_heap_t,
-        size: usize,
-        alignment: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_zalloc_aligned_at`], but allocates out of the
-    /// specific heap instead of the default.
-    pub fn mi_heap_zalloc_aligned_at(
-        heap: *mut mi_heap_t,
-        size: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_calloc_aligned`], but allocates out of the specific
-    /// heap instead of the default.
-    pub fn mi_heap_calloc_aligned(
-        heap: *mut mi_heap_t,
-        count: usize,
-        size: usize,
-        alignment: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_calloc_aligned_at`], but allocates out of the
-    /// specific heap instead of the default.
-    pub fn mi_heap_calloc_aligned_at(
-        heap: *mut mi_heap_t,
-        count: usize,
-        size: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_realloc_aligned`](crate::mi_realloc_aligned), but allocates out of the specific
-    /// heap instead of the default.
-    pub fn mi_heap_realloc_aligned(
-        heap: *mut mi_heap_t,
-        p: *mut c_void,
-        newsize: usize,
-        alignment: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_realloc_aligned_at`], but allocates out of the
-    /// specific heap instead of the default.
-    pub fn mi_heap_realloc_aligned_at(
-        heap: *mut mi_heap_t,
-        p: *mut c_void,
-        newsize: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_rezalloc`], but allocates out of the specific heap
-    /// instead of the default.
-    pub fn mi_heap_rezalloc(heap: *mut mi_heap_t, p: *mut c_void, newsize: usize) -> *mut c_void;
-
-    /// Equivalent to [`mi_recalloc`], but allocates out of the specific heap
-    /// instead of the default.
-    pub fn mi_heap_recalloc(
-        heap: *mut mi_heap_t,
-        p: *mut c_void,
-        newcount: usize,
-        size: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_rezalloc_aligned`], but allocates out of the specific
-    /// heap instead of the default.
-    pub fn mi_heap_rezalloc_aligned(
-        heap: *mut mi_heap_t,
-        p: *mut c_void,
-        newsize: usize,
-        alignment: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_rezalloc_aligned_at`], but allocates out of the
-    /// specific heap instead of the default.
-    pub fn mi_heap_rezalloc_aligned_at(
-        heap: *mut mi_heap_t,
-        p: *mut c_void,
-        newsize: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_recalloc_aligned`], but allocates out of the
-    /// specific heap instead of the default.
-    pub fn mi_heap_recalloc_aligned(
-        heap: *mut mi_heap_t,
-        p: *mut c_void,
-        newcount: usize,
-        size: usize,
-        alignment: usize,
-    ) -> *mut c_void;
-
-    /// Equivalent to [`mi_recalloc_aligned_at`], but allocates out of the
-    /// specific heap instead of the default.
-    pub fn mi_heap_recalloc_aligned_at(
-        heap: *mut mi_heap_t,
-        p: *mut c_void,
-        newcount: usize,
-        size: usize,
-        alignment: usize,
-        offset: usize,
-    ) -> *mut c_void;
-
-    /// Does a heap contain a pointer to a previously allocated block?
-    ///
-    /// `p` must be a pointer to a previously allocated block (in any heap) -- it cannot be some
-    /// random pointer!
-    ///
-    /// Returns `true` if the block pointed to by `p` is in the `heap`.
-    ///
-    /// See [`mi_heap_check_owned`].
-    #[cfg(feature = "v2")]
-    pub fn mi_heap_contains_block(heap: *mut mi_heap_t, p: *const c_void) -> bool;
-
-    /// Check safely if any pointer is part of a heap.
-    ///
-    /// `p` may be any pointer -- not required to be previously allocated by the
-    /// given heap or any other mimalloc heap. Returns `true` if `p` points to a
-    /// block in the given heap, false otherwise.
-    ///
-    /// Note: expensive function, linear in the pages in the heap.
-    ///
-    /// See [`mi_heap_contains_block`], [`mi_heap_get_default`], and
-    /// [`mi_is_in_heap_region`]
-    #[cfg(feature = "v2")]
-    pub fn mi_heap_check_owned(heap: *mut mi_heap_t, p: *const c_void) -> bool;
-
-    /// Check safely if any pointer is part of the default heap of this thread.
-    ///
-    /// `p` may be any pointer -- not required to be previously allocated by the
-    /// default heap for this thread, or any other mimalloc heap. Returns `true`
-    /// if `p` points to a block in the default heap, false otherwise.
-    ///
-    /// Note: expensive function, linear in the pages in the heap.
-    ///
-    /// See [`mi_heap_contains_block`], [`mi_heap_get_default`]
-    pub fn mi_check_owned(p: *const c_void) -> bool;
-
-    /// Visit all areas and blocks in `heap`.
-    ///
-    /// If `visit_all_blocks` is false, the `visitor` is only called once for
-    /// every heap area. If it's true, the `visitor` is also called for every
-    /// allocated block inside every area (with `!block.is_null()`). Return
-    /// `false` from the `visitor` to return early.
-    ///
-    /// `arg` is an extra argument passed into the `visitor`.
-    ///
-    /// Returns `true` if all areas and blocks were visited.
-    ///
-    /// Passing a `None` visitor is allowed, and is a no-op.
-    #[cfg(feature = "v2")]
-    pub fn mi_heap_visit_blocks(
-        heap: *const mi_heap_t,
-        visit_all_blocks: bool,
-        visitor: mi_block_visit_fun,
-        arg: *mut c_void,
-    ) -> bool;
-
-    /// Visit all areas and blocks in `heap`.
-    ///
-    /// If `visit_all_blocks` is false, the `visitor` is only called once for
-    /// every heap area. If it's true, the `visitor` is also called for every
-    /// allocated block inside every area (with `!block.is_null()`). Return
-    /// `false` from the `visitor` to return early.
-    ///
-    /// `arg` is an extra argument passed into the `visitor`.
-    ///
-    /// Returns `true` if all areas and blocks were visited.
-    ///
-    /// Passing a `None` visitor is allowed, and is a no-op.
-    ///
-    /// Note: in v3 the `heap` parameter is non-const compared to v2.
-    #[cfg(not(feature = "v2"))]
-    pub fn mi_heap_visit_blocks(
-        heap: *mut mi_heap_t,
-        visit_all_blocks: bool,
-        visitor: mi_block_visit_fun,
-        arg: *mut c_void,
-    ) -> bool;
-
-    #[cfg(feature = "arena")]
-    /// Create a heap that only allocates in the specified arena
-    pub fn mi_heap_new_in_arena(arena_id: mi_arena_id_t) -> *mut mi_heap_t;
-
-    #[cfg(feature = "arena")]
-    /// Reserve OS memory for use by mimalloc. Reserved areas are used
-    /// before allocating from the OS again. By reserving a large area upfront,
-    /// allocation can be more efficient, and can be better managed on systems
-    /// without `mmap`/`VirtualAlloc` (like WASM for example).
-    ///
-    /// - `size` The size to reserve.
-    /// - `commit` Commit the memory upfront.
-    /// - `allow_large` Allow large OS pages (2MiB) to be used?
-    /// - `exclusive` Only allow allocations if specifically for this arena.
-    /// - `arena_id` Pointer who's value will be set to the new arena_id if successful.
-    ///
-    /// Returns 0 if successful, and an error code otherwise (e.g. `ENOMEM`)
-    pub fn mi_reserve_os_memory_ex(
-        size: usize,
-        commit: bool,
-        allow_large: bool,
-        exclusive: bool,
-        arena_id: *mut mi_arena_id_t,
-    ) -> c_int;
-
-    /// Check if the heap page containing `p` is under-utilized.
-    ///
-    /// # Safety
-    /// Assumes the page belonging to `p` is only accessed by the calling thread.
-    pub fn mi_unsafe_heap_page_is_under_utilized(
-        heap: *mut mi_heap_t,
-        p: *mut c_void,
-        perc_threshold: usize,
-    ) -> bool;
-
-    #[cfg(not(feature = "v2"))]
-    /// Return the minimum size for an arena (v3 only).
-    pub fn mi_arena_min_size() -> usize;
-
-    #[cfg(not(feature = "v2"))]
-    /// Equivalent to [`mi_heap_zalloc_small`], but for a thread-local heap (`theap`) in v3.
-    ///
-    /// `size` must be smaller or equal to [`MI_SMALL_SIZE_MAX`].
-    pub fn mi_theap_zalloc_small(theap: *mut mi_theap_t, size: usize) -> *mut c_void;
-
-    #[cfg(feature = "arena")]
-    /// Manage a particular memory area for use by mimalloc.
-    /// This is just like `mi_reserve_os_memory_ex` except that the area should already be
-    /// allocated in some manner and available for use my mimalloc.
-    ///
-    /// # Safety
-    /// mimalloc will likely segfault when allocating from the arena if the arena `start` & `size`
-    /// aren't aligned with mimalloc's `MI_SEGMENT_ALIGN` (e.g. 32MB on x86_64 machines).
-    ///
-    /// - `start` Start of the memory area
-    /// - `size` The size of the memory area. Must be large than `MI_ARENA_BLOCK_SIZE` (e.g. 64MB
-    ///          on x86_64 machines).
-    /// - `commit` Set true if the memory range is already commited.
-    /// - `is_large` Set true if the memory range consists of large files, or if the memory should
-    ///              not be decommitted or protected (like rdma etc.).
-    /// - `is_zero` Set true if the memory range consists only of zeros.
-    /// - `numa_node` Possible associated numa node or `-1`.
-    /// - `exclusive` Only allow allocations if specifically for this arena.
-    /// - `arena_id` Pointer who's value will be set to the new arena_id if successful.
-    ///
-    /// Returns `true` if arena was successfully allocated
-    pub fn mi_manage_os_memory_ex(
-        start: *const c_void,
-        size: usize,
-        is_committed: bool,
-        is_large: bool,
-        is_zero: bool,
-        numa_node: c_int,
-        exclusive: bool,
-        arena_id: *mut mi_arena_id_t,
-    ) -> bool;
-}
-
-#[cfg(test)]
-mod tests {
-    use super::super::mi_malloc;
-    use super::*;
-
-    #[test]
-    fn it_calculates_usable_size() {
-        let ptr = unsafe { mi_malloc(32) } as *mut u8;
-        let usable_size = unsafe { mi_usable_size(ptr as *mut c_void) };
-        assert!(
-            usable_size >= 32,
-            "usable_size should at least equal to the allocated size"
-        );
-    }
-
-    #[test]
-    fn runtime_stable_option() {
-        unsafe {
-            assert_eq!(mi_option_get(mi_option_show_errors), 0);
-            mi_option_set(mi_option_show_errors, 1);
-            assert_eq!(mi_option_get(mi_option_show_errors), 1);
-
-            assert_eq!(mi_option_get(mi_option_show_stats), 0);
-            mi_option_set(mi_option_show_stats, 1);
-            assert_eq!(mi_option_get(mi_option_show_stats), 1);
-
-            assert_eq!(mi_option_get(mi_option_verbose), 0);
-            mi_option_set(mi_option_verbose, 1);
-            assert_eq!(mi_option_get(mi_option_verbose), 1);
-        }
-    }
-}
diff --git a/vendor/libmimalloc-sys/src/lib.rs b/vendor/libmimalloc-sys/src/lib.rs
deleted file mode 100644
index ebd069b4..00000000
--- a/vendor/libmimalloc-sys/src/lib.rs
+++ /dev/null
@@ -1,139 +0,0 @@
-#![no_std]
-// Copyright 2019 Octavian Oncescu
-
-use core::ffi::c_void;
-
-#[cfg(feature = "extended")]
-mod extended;
-#[cfg(feature = "extended")]
-pub use extended::*;
-
-extern "C" {
-    /// Allocate zero-initialized `size` bytes.
-    ///
-    /// Returns a pointer to newly allocated zero-initialized memory, or null if
-    /// out of memory.
-    pub fn mi_zalloc(size: usize) -> *mut c_void;
-
-    /// Allocate `size` bytes.
-    ///
-    /// Returns pointer to the allocated memory or null if out of memory.
-    /// Returns a unique pointer if called with `size` 0.
-    pub fn mi_malloc(size: usize) -> *mut c_void;
-
-    /// Re-allocate memory to `newsize` bytes.
-    ///
-    /// Return pointer to the allocated memory or null if out of memory. If null
-    /// is returned, the pointer `p` is not freed. Otherwise the original
-    /// pointer is either freed or returned as the reallocated result (in case
-    /// it fits in-place with the new size).
-    ///
-    /// If `p` is null, it behaves as [`mi_malloc`]. If `newsize` is larger than
-    /// the original `size` allocated for `p`, the bytes after `size` are
-    /// uninitialized.
-    pub fn mi_realloc(p: *mut c_void, newsize: usize) -> *mut c_void;
-
-    /// Allocate `size` bytes aligned by `alignment`, initialized to zero.
-    ///
-    /// Return pointer to the allocated memory or null if out of memory.
-    ///
-    /// Returns a unique pointer if called with `size` 0.
-    pub fn mi_zalloc_aligned(size: usize, alignment: usize) -> *mut c_void;
-
-    /// Allocate `size` bytes aligned by `alignment`.
-    ///
-    /// Return pointer to the allocated memory or null if out of memory.
-    ///
-    /// Returns a unique pointer if called with `size` 0.
-    pub fn mi_malloc_aligned(size: usize, alignment: usize) -> *mut c_void;
-
-    /// Re-allocate memory to `newsize` bytes, aligned by `alignment`.
-    ///
-    /// Return pointer to the allocated memory or null if out of memory. If null
-    /// is returned, the pointer `p` is not freed. Otherwise the original
-    /// pointer is either freed or returned as the reallocated result (in case
-    /// it fits in-place with the new size).
-    ///
-    /// If `p` is null, it behaves as [`mi_malloc_aligned`]. If `newsize` is
-    /// larger than the original `size` allocated for `p`, the bytes after
-    /// `size` are uninitialized.
-    pub fn mi_realloc_aligned(p: *mut c_void, newsize: usize, alignment: usize) -> *mut c_void;
-
-    /// Free previously allocated memory.
-    ///
-    /// The pointer `p` must have been allocated before (or be null).
-    pub fn mi_free(p: *mut c_void);
-}
-
-/// When using the `"override"` feature flag, the user wants us to globally
-/// override the system allocator.
-///
-/// However, since we build and link `mimalloc` as a static library/archive,
-/// the linker may decide to not care about our overrides if it can't directly
-/// see references to the symbols, see the following link for details:
-/// <https://maskray.me/blog/2021-06-20-symbol-processing#archive-processing>
-///
-/// This is problematic if `mimalloc` is used from a library that by itself
-/// doesn't allocate, yet invokes other shared libraries that do, since then
-/// the linker wouldn't see any references to `malloc`/`free`, and the symbols
-/// would not be overridden.
-///
-/// To avoid this problem, we make sure that the allocator functions are
-/// visible to the linker.
-///
-/// To avoid this problem, we reference `mi_malloc` in a `#[used]` static.
-/// This makes it known to `rustc`, which will create a reference to it in a
-/// `symbols.o` stub file that is later passed directly to the linker (instead
-/// of being in an archive). See the following link for details on how this
-/// works: <https://github.com/rust-lang/rust/pull/95604>
-///
-/// NOTE: This works because `mimalloc` is compiled into a single object file
-/// in `static.c`. If it was split across multiple files, we'd need to
-/// reference each symbol. See also the comment at the top of `static.c`.
-///
-/// NOTE: On macOS, mimalloc doesn't just override malloc/free, but also
-/// registers itself with the allocator's zone APIs in a ctor
-/// (`_mi_macos_override_malloc`, marked with `__attribute__((constructor))`).
-/// Similarly to above, for the Mach-O linker to actually consider ctors as
-/// "used" when defined in an archive member in a static library, so we need
-/// to explicitly reference something in the object file. The constructor
-/// symbol itself is static, so we can't get a reference to that, so instead
-/// we reference `mi_malloc` here too).
-#[cfg(feature = "override")]
-mod set_up_statics {
-    use super::*;
-    #[used] // Could be `#[used(linker)]` once stable
-    static USED: unsafe extern "C" fn(usize) -> *mut c_void = mi_malloc;
-}
-
-#[cfg(test)]
-mod tests {
-    extern crate libc;
-    use super::*;
-
-    #[test]
-    fn it_frees_memory_malloc() {
-        let ptr = unsafe { mi_malloc_aligned(8, 8) } as *mut u8;
-        unsafe { mi_free(ptr as *mut c_void) };
-    }
-
-    #[test]
-    fn it_frees_memory_zalloc() {
-        let ptr = unsafe { mi_zalloc_aligned(8, 8) } as *mut u8;
-        unsafe { mi_free(ptr as *mut c_void) };
-    }
-
-    #[test]
-    fn it_frees_memory_realloc() {
-        let ptr = unsafe { mi_malloc_aligned(8, 8) } as *mut u8;
-        let ptr = unsafe { mi_realloc_aligned(ptr as *mut c_void, 8, 8) } as *mut u8;
-        unsafe { mi_free(ptr as *mut c_void) };
-    }
-
-    #[cfg(all(feature = "override", target_vendor = "apple"))]
-    #[test]
-    fn mimalloc_and_libc_are_interoperable_when_overridden() {
-        let ptr = unsafe { mi_malloc(42) };
-        unsafe { libc::free(ptr) };
-    }
-}

From 2b174f2d77c9608da73f28358caf067f152d5237 Mon Sep 17 00:00:00 2001
From: Owen Carey <37121709+owenthcarey@users.noreply.github.com>
Date: Mon, 19 Oct 2026 23:53:05 -0700
Subject: [PATCH 173/173] perf: skip the empty memset when clearing a small
 dict table

`Table::clear` filled the slot table with zeros even when a small table
has none, so a recycled keyword dict made a zero-byte `memset` call at
the empty slice's dangling address on every `f(x, name=value)` call into
a `**kwargs` function. On AMD Zen 4 and Zen 5 that call cost about 100 ns
(glibc's AVX-512 `memset` stores through a mask there, apparently taking
a microcode assist for the unmapped page), which made the bench gate's
call_overhead fixture 1.5x slower than main on those runners. Probed on
EPYC 9V45 runners: call_overhead goes from 49 ms to 31 ms (main: 31 ms).
---
 crates/weavepy-vm/src/dictmap.rs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/crates/weavepy-vm/src/dictmap.rs b/crates/weavepy-vm/src/dictmap.rs
index c9b1198e..ea6c9e4b 100644
--- a/crates/weavepy-vm/src/dictmap.rs
+++ b/crates/weavepy-vm/src/dictmap.rs
@@ -311,9 +311,18 @@ impl<V> Table<V> {
     }
 
     /// Remove every entry, keeping the allocations.
+    #[inline]
     pub fn clear(&mut self) {
         self.entries.clear();
-        self.slots.fill(0);
+        // A small table has no slots, so skip the `memset` call. A recycled
+        // keyword dict otherwise made one per call, of zero bytes at the
+        // empty slice's dangling address, and on AMD Zen 4 and Zen 5 that
+        // cost about 100 ns (glibc's AVX-512 `memset` stores through a mask
+        // there, apparently taking a microcode assist for the unmapped
+        // page).
+        if !self.slots.is_empty() {
+            self.slots.fill(0);
+        }
     }
 
     /// Rebuild the slot table at `n` slots (a power of two) from the